@inproceedings{neidle:22037:sign-lang:lrec, author = {Neidle, Carol and Opoku, Augustine and Ballard, Carey M. and Dafnis, Konstantinos M. and Chroni, Evgenia and Metaxas, Dimitris}, title = {Resources for Computer-Based Sign Recognition from Video, and the Criticality of Consistency of Gloss Labeling across Multiple Large {ASL} Video Corpora}, pages = {165--172}, editor = {Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and Hochgesang, Julie A. and Kristoffersen, Jette and Mesch, Johanna and Schulder, Marc}, booktitle = {Proceedings of the {LREC2022} 10th Workshop on the Representation and Processing of Sign Languages: Multilingual Sign Language Resources}, maintitle = {13th International Conference on Language Resources and Evaluation ({LREC} 2022)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Marseille, France}, day = {25}, month = jun, year = {2022}, isbn = {979-10-95546-86-3}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/22037.html}, abstract = {The WLASL purports to be ``the largest video dataset for Word-Level American Sign Language (ASL) recognition.'' It brings together various publicly shared video collections that could be quite valuable for sign recognition research, and it has been used extensively for such research. However, a critical problem with the accompanying annotations has heretofore not been recognized by the authors, nor by those who have exploited these data: There is no 1-1 correspondence between sign productions and gloss labels. Here we describe a large (and recently expanded and enhanced), linguistically annotated, downloadable, video corpus of citation-form ASL signs shared by the American Sign Language Linguistic Research Project (ASLLRP)---with 23,452 sign tokens and an online Sign Bank---in which such correspondences are enforced. We furthermore provide annotations for 19,672 of the WLASL video examples consistent with ASLLRP glossing conventions. For those wishing to use WLASL videos, this provides a set of annotations that makes it possible: (1) to use those data reliably for computational research; and/or (2) to combine the WLASL and ASLLRP datasets, creating a combined resource that is larger and richer than either of those datasets individually, with consistent gloss labeling for all signs. We also offer a summary of our own sign recognition research to date that exploits these data resources.} } @inproceedings{neidle:18001:sign-lang:lrec, author = {Neidle, Carol and Opoku, Augustine and Dimitriadis, Gregory and Metaxas, Dimitris}, title = {New shared {\&} interconnected {ASL} resources: {SignStream{\textregistered}} 3 software; {DAI} 2 for web access to linguistically annotated video corpora; and a sign bank}, pages = {147--154}, editor = {Bono, Mayumi and Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and Hochgesang, Julie A. and Kristoffersen, Jette and Mesch, Johanna and Osugi, Yutaka}, booktitle = {Proceedings of the {LREC2018} 8th Workshop on the Representation and Processing of Sign Languages: Involving the Language Community}, maintitle = {11th International Conference on Language Resources and Evaluation ({LREC} 2018)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Miyazaki, Japan}, day = {12}, month = may, year = {2018}, isbn = {979-10-95546-01-6}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/18001.html}, abstract = {2017 marked the release of a new version of SignStream{\textregistered} software, designed to facilitate linguistic analysis of ASL video. SignStream{\textregistered} provides an intuitive interface for labeling and time-aligning manual and non-manual components of the signing. Version 3 has many new features. For example, it enables representation of morpho-phonological information, including display of handshapes. An expanding ASL video corpus, annotated through use of SignStream{\textregistered}, is shared publicly on the Web. This corpus (video plus annotations) is Web-accessible---browsable, searchable, and downloadable---thanks to a new, improved version of our Data Access Interface: DAI 2. DAI 2 also offers Web access to a brand new Sign Bank, containing about 10,000 examples of about 3,000 distinct signs, as produced by up to 9 different ASL signers. This Sign Bank is also directly accessible from within SignStream{\textregistered}, thereby boosting the efficiency and consistency of annotation; new items can also be added to the Sign Bank. Soon to be integrated into SignStream{\textregistered} 3 and DAI 2 are visualizations of computer-generated analyses of the video: graphical display of eyebrow height, eye aperture, and head position. These resources are publicly available, for linguistic and computational research and for those who use or study ASL.} } @inproceedings{neidle:14004:sign-lang:lrec, author = {Neidle, Carol and Liu, Jingjing and Liu, Bo and Peng, Xi and Vogler, Christian and Metaxas, Dimitris}, title = {Computer-based tracking, analysis, and visualization of linguistically significant non-manual events in {American} {Sign} {Language} ({ASL})}, pages = {127--134}, editor = {Crasborn, Onno and Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and Hochgesang, Julie A. and Kristoffersen, Jette and Mesch, Johanna}, booktitle = {Proceedings of the {LREC2014} 6th Workshop on the Representation and Processing of Sign Languages: Beyond the Manual Channel}, maintitle = {9th International Conference on Language Resources and Evaluation ({LREC} 2014)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Reykjavik, Iceland}, day = {31}, month = may, year = {2014}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/14004.html}, abstract = {Our linguistically annotated American Sign Language (ASL) corpora have formed a basis for research to automate detection by computer of essential linguistic information conveyed through facial expressions and head movements. We have tracked head position and facial deformations, and used computational learning to discern specific grammatical markings. Our ability to detect, identify, and temporally localize the occurrence of such markings in ASL videos has recently been improved by incorporation of (1) new techniques for deformable model-based 3D tracking of head position and facial expressions, which provide significantly better tracking accuracy and recover quickly from temporary loss of track due to occlusion; and (2) a computational learning approach incorporating 2-level Conditional Random Fields (CRFs), suited to the multi-scale spatio-temporal characteristics of the data, which analyses not only low-level appearance characteristics, but also the patterns that enable identification of significant gestural components, such as periodic head movements and raised or lowered eyebrows. Here we summarize our linguistically motivated computational approach and the results for detection and recognition of nonmanual grammatical markings; demonstrate our data visualizations, and discuss the relevance for linguistic research; and describe work underway to enable such visualizations to be produced over large corpora and shared publicly on the Web.} } @inproceedings{liu-etal-2014-3d:lrec, author = {Liu, Bo and Liu, Jingjing and Yu, Xiang and Metaxas, Dimitris and Neidle, Carol}, title = {3{D} Face Tracking and Multi-Scale, Spatio-temporal Analysis of Linguistically Significant Facial Expressions and Head Positions in {ASL}}, pages = {4512--4518}, editor = {Calzolari, Nicoletta and Choukri, Khalid and Declerck, Thierry and Loftsson, Hrafn and Maegaard, Bente and Mariani, Joseph and Moreno, Asuncion and Odijk, Jan and Piperidis, Stelios,}, booktitle = {9th International Conference on Language Resources and Evaluation ({LREC} 2014)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Reykjavik, Iceland}, day = {26--31}, month = may, year = {2014}, isbn = {978-2-9517408-8-4}, language = {english}, url = {https://aclanthology.org/L14-1318}, abstract = {Essential grammatical information is conveyed in signed languages by clusters of events involving facial expressions and movements of the head and upper body. This poses a significant challenge for computer-based sign language recognition. Here, we present new methods for the recognition of nonmanual grammatical markers in American Sign Language (ASL) based on: (1) new 3D tracking methods for the estimation of 3D head pose and facial expressions to determine the relevant low-level features; (2) methods for higher-level analysis of component events (raised/lowered eyebrows, periodic head nods and head shakes) used in grammatical markings―with differentiation of temporal phases (onset, core, offset, where appropriate), analysis of their characteristic properties, and extraction of corresponding features; (3) a 2-level learning framework to combine low- and high-level features of differing spatio-temporal scales. This new approach achieves significantly better tracking and recognition results than our previous methods.} } @inproceedings{neidle:12027:sign-lang:lrec, author = {Neidle, Carol and Vogler, Christian}, title = {A New Web Interface to Facilitate Access to Corpora: Development of the {ASLLRP} Data Access Interface}, pages = {137--142}, editor = {Crasborn, Onno and Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and Kristoffersen, Jette and Mesch, Johanna}, booktitle = {Proceedings of the {LREC2012} 5th Workshop on the Representation and Processing of Sign Languages: Interactions between Corpus and Lexicon}, maintitle = {8th International Conference on Language Resources and Evaluation ({LREC} 2012)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Istanbul, Turkey}, day = {27}, month = may, year = {2012}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/12027.html}, abstract = {A significant obstacle to broad utilization of corpora is the difficulty in gaining access to the specific subsets of data and annotations that may be relevant for particular types of research. With that in mind, we have developed a web-based Data Access Interface (DAI), to provide access to the expanding datasets of the American Sign Language Linguistic Research Project (ASLLRP). The DAI facilitates browsing the corpora, viewing videos and annotations, searching for phenomena of interest, and downloading selected materials from the website. The web interface, compared to providing videos and annotation files off-line, also greatly increases access by people that have no prior experience in working with linguistic annotation tools, and it opens the door to integrating the data with third-party applications on the desktop and in the mobile space. In this paper we give an overview of the available videos, annotations, and search functionality of the DAI, as well as plans for future enhancements. We also summarize best practices and key lessons learned that are crucial to the success of similar projects.} } @inproceedings{metaxas-etal-2012-recognition:lrec, author = {Metaxas, Dimitris and Liu, Bo and Yang, Fei and Yang, Peng and Michael, Nicholas and Neidle, Carol}, title = {Recognition of Nonmanual Markers in {A}merican {S}ign {L}anguage ({ASL}) Using Non-Parametric Adaptive 2{D}-3{D} Face Tracking}, pages = {2414--2420}, editor = {Calzolari, Nicoletta and Choukri, Khalid and Declerck, Thierry and Do{\u g}an, Mehmet U{\u g}ur and Maegaard, Bente and Mariani, Joseph and Moreno, Asuncion and Odijk, Jan and Piperidis, Stelios}, booktitle = {8th International Conference on Language Resources and Evaluation ({LREC} 2012)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Istanbul, Turkey}, day = {21--27}, month = may, year = {2012}, isbn = {978-2-9517408-7-7}, language = {english}, url = {https://aclanthology.org/L12-1123}, abstract = {This paper addresses the problem of automatically recognizing linguistically significant nonmanual expressions in American Sign Language from video. We develop a fully automatic system that is able to track facial expressions and head movements, and detect and recognize facial events continuously from video. The main contributions of the proposed framework are the following: (1) We have built a stochastic and adaptive ensemble of face trackers to address factors resulting in lost face track; (2) We combine 2D and 3D deformable face models to warp input frames, thus correcting for any variation in facial appearance resulting from changes in 3D head pose; (3) We use a combination of geometric features and texture features extracted from a canonical frontal representation. The proposed new framework makes it possible to detect grammatically significant nonmanual expressions from continuous signing and to differentiate successfully among linguistically significant expressions that involve subtle differences in appearance. We present results that are based on the use of a dataset containing 330 sentences from videos that were collected and linguistically annotated at Boston University.} } @inproceedings{athitsos:10022:sign-lang:lrec, author = {Athitsos, Vassilis and Neidle, Carol and Sclaroff, Stan and Nash, Joan and Stefan, Alexandra and Thangali, Ashwin and Wang, Haijing and Yuan, Quan}, title = {Large Lexicon Project: {American} {Sign} {Language} Video Corpus and Sign Language Indexing/Retrieval Algorithms}, pages = {11--14}, editor = {Dreuw, Philippe and Efthimiou, Eleni and Hanke, Thomas and Johnston, Trevor and Mart{\'i}nez Ruiz, Gregorio and Schembri, Adam}, booktitle = {Proceedings of the {LREC2010} 4th Workshop on the Representation and Processing of Sign Languages: Corpora and Sign Language Technologies}, maintitle = {7th International Conference on Language Resources and Evaluation ({LREC} 2010)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Valletta, Malta}, day = {22--23}, month = may, year = {2010}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/10022.html}, abstract = {When we encounter a word that we do not understand in a written language, we can look it up in a dictionary. However, looking up the meaning of an unknown sign in American Sign Language (ASL) is not nearly as straightforward. This paper describes progress in an ongoing project aiming to build a computer vision system that helps users look up the meaning of an unknown ASL sign. When a user encounters an unknown ASL sign, the user submits a video of that sign as a query to the system. The system evaluates the similarity between the query and video examples of all signs in the known lexicon, and presents the most similar signs to the user. The user can then look at the retrieved signs and determine if any of them matches the query sign. \par An important part of the project is building a video database containing examples of a large number of signs. So far we have recorded at least two video examples for almost all of the 3,000 signs contained in the Gallaudet dictionary. Each video sequence is captured simultaneously from four different cameras, providing two frontal views, a side view, and a view zoomed in on the face of the signer. Our entire video dataset is publicly available on the Web. \par Automatic computer vision-based evaluation of similarity between signs is a challenging task. In order to improve accuracy, we manually annotate the hand locations in each frame of each sign in the database. While this is a time-consuming process, this process incurs a one-time preprocessing cost that is invisible to the end-user of the system. At runtime, once the user has submitted the query video, the current version of the system asks the user to specify hand locations in the first frame, and then the system automatically tracks the location of the hands in the rest of the query video. The user can review and correct the hand location results. Every correction that the user makes on a specific frame is used by the system to further improve the hand location estimates in other frames. \par Once hand locations have been estimated for the query video, the system evaluates the similarity between the query video and every sign video in the database. Similarity is measured using the Dynamic Time Warping (DTW) algorithm, a well-known algorithm for comparing time series. The performance of the system has been evaluated in experiments where 933 signs from 921 distinct sign classes are used as the dataset of known signs, and 193 signs are used as a test set. In those experiments, only a single frontal view was used for all test and training examples. For 68{\%} of the test signs, the correct sign is included in the 20 most similar signs retrieved by the system. \par In ongoing work, we are manually annotating hand locations in the remainder of our collected videos, so as to gradually incorporate more signs into our system. We are also investigating better ways for measuring similarity between signs, and for making the system more automatic, reducing or eliminating the need for the user to manually provide information to the system about hand locations.} } @inproceedings{michael:10029:sign-lang:lrec, author = {Michael, Nicholas and Neidle, Carol and Metaxas, Dimitris}, title = {Computer-based recognition of facial expressions in {ASL}: from face tracking to linguistic interpretation}, pages = {164--167}, editor = {Dreuw, Philippe and Efthimiou, Eleni and Hanke, Thomas and Johnston, Trevor and Mart{\'i}nez Ruiz, Gregorio and Schembri, Adam}, booktitle = {Proceedings of the {LREC2010} 4th Workshop on the Representation and Processing of Sign Languages: Corpora and Sign Language Technologies}, maintitle = {7th International Conference on Language Resources and Evaluation ({LREC} 2010)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Valletta, Malta}, day = {22--23}, month = may, year = {2010}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/10029.html}, abstract = {Most research in the field of sign language recognition has focused on the manual component of signing, despite the fact that there is critical grammatical information expressed through facial expressions and head gestures. We, therefore, propose a novel framework for robust tracking and analysis of nonmanual behaviors, with an application to sign language recognition. \par Our method uses computer vision techniques to track facial expressions and head movements from video, in order to recognize such linguistically significant expressions. The methods described here have relied crucially on the use of a linguistically annotated video corpus that is being developed, as the annotated video examples have served for training and testing our machine learning models. We apply our framework to continuous recognition of three classes of grammatical expressions, namely wh-questions, negative expressions, and topics. \par Our method is signer-independent, utilizing spatial pyramids and Hidden Markov Models (HMMs) to model the temporal variations of facial shape and appearance.} }