In sign languages, syllables are composed of syllabic components consisting of locations, movements, and handshapes; however, the rules of combinations of these syllabic components are still unclear. Decomposing existing syllables into syllabic components is necessary to clarify the rules. This study aims to construct an automatic syllabic component classification system for Japanese Sign Language (JSL) using deep learning. We propose a pre-training method using non-Japanese Sign Language data to achieve high performance in classifying syllabic components in a situation where the number of training JSL videos is limited. We also investigate multitask learning for syllabic component classification to share the information among the syllabic components. Experiments on the syllabic component classification for the dominant hand show that 1) pre-training with the American Sign Language (ASL) dataset improved classification performance for the movement and handshape components and 2) multitask learning did not contribute to the performance improvement of syllabic component classification. We also investigated the influence of pre-training on syllabic component classification by visualizing critical elements in videos to predict the components.
@inproceedings{inoue:24022:sign-lang:lrec,
author = {Inoue, Jundai and Miwa, Makoto and Sasaki, Yutaka and Hara, Daisuke},
title = {Enhancing Syllabic Component Classification in {Japanese} {Sign} {Language} by Pre-training on Non-Japanese Sign Language Data},
pages = {181--188},
editor = {Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and Hochgesang, Julie A. and Mesch, Johanna and Schulder, Marc},
booktitle = {Proceedings of the {LREC-COLING} 2024 11th Workshop on the Representation and Processing of Sign Languages: Evaluation of Sign Language Resources},
maintitle = {2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation ({LREC-COLING} 2024)},
publisher = {{ELRA Language Resources Association (ELRA) and the International Committee on Computational Linguistics (ICCL)}},
address = {Torino, Italy},
day = {25},
month = may,
year = {2024},
isbn = {978-2-493814-30-2},
language = {english},
url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/24022.pdf}
}
| Sasaki, Yutaka