To improve computer-based recognition from video of isolated signs from American Sign Language (ASL), we propose a new skeleton-based method that involves explicit detection of the start and end frames of signs, trained on the ASLLVD dataset; it uses linguistically relevant parameters based on the skeleton input. Our method employs a bidirectional learning approach within a Graph Convolutional Network (GCN) framework. We apply this method to the WLASL dataset, but with corrections to the gloss labeling to ensure consistency in the labels assigned to different signs; it is important to have a 1-1 correspondence between signs and text-based gloss labels. We achieve a success rate of 77.43% for top-1 and 94.54% for top-5 using this modified WLASL dataset. Our method, which does not require multi-modal data input, outperforms other state-of-the-art approaches on the same modified WLASL dataset, demonstrating the importance of both attention to the start and end frames of signs and the use of bidirectional data streams in the GCNs for isolated sign recognition.
@inproceedings{dafnis-etal-2022-bidirectional:lrec,
author = {Dafnis, Konstantinos M. and Chroni, Evgenia and Neidle, Carol and Metaxas, Dimitri},
title = {Bidirectional Skeleton-Based Isolated Sign Recognition using Graph Convolution Networks and Transfer Learning},
pages = {7328--7338},
editor = {Calzolari, Nicoletta and Fr{\'e}d{\'e}ric B{\'e}chet and Blache, Philippe and Choukri, Khalid and Cieri, Christopher and Declerck, Thierry and Goggi, Sara and Isahara, Hitoshi and Maegaard, Bente and Mariani, Joseph and Mazo, H{\'e}l{\`e}ne and Odijk, Jan and Piperidis, Stelios},
booktitle = {13th International Conference on Language Resources and Evaluation ({LREC} 2022)},
publisher = {{European Language Resources Association (ELRA)}},
address = {Marseille, France},
day = {20--25},
month = jun,
year = {2022},
isbn = {979-10-95546-72-6},
language = {english},
url = {http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.797}
}