@inproceedings{sun-etal-2024-adaptive:lrec, author = {Sun, Tong and Fu, Biao and Hu, Cong and Zhang, Liang and Zhang, Ruiquan and Shi, Xiaodong and Su, Jinsong and Chen, Yidong}, title = {Adaptive Simultaneous Sign Language Translation with Confident Translation Length Estimation}, pages = {372--384}, editor = {Calzolari, Nicoletta and Kan, Min-Yen and Hoste, Veronique and Lenci, Alessandro and Sakti, Sakriani and Xue, Nianwen}, booktitle = {2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation ({LREC-COLING} 2024)}, publisher = {{ELRA Language Resources Association (ELRA) and the International Committee on Computational Linguistics (ICCL)}}, address = {Torino, Italy}, day = {20--25}, month = may, year = {2024}, isbn = {978-2-493814-10-4}, language = {english}, url = {https://aclanthology.org/2024.lrec-main.34}, abstract = {Traditional non-simultaneous Sign Language Translation (SLT) methods, while effective for pre-recorded videos, face challenges in real-time scenarios due to inherent inference delays. The emerging field of simultaneous SLT aims to address this issue by progressively translating incrementally received sign video. However, the sole existing work in simultaneous SLT adopts a fixed gloss-based policy, which suffer from limitations in boundary prediction and contextual comprehension. In this paper, we delve deeper into this area and propose an adaptive policy for simultaneous SLT. Our approach introduces the concept of ``confident translation length'', denoting maximum accurate translation achievable from current input. An estimator measures this length for streaming sign video, enabling the model to make informed decisions on whether to wait for more input or proceed with translation. To train the estimator, we construct a training data of confident translation length based on the longest common prefix between translations of partial and complete inputs. Furthermore, we incorporate adaptive training, utilizing pseudo prefix pairs, to refine the offline translation model for optimal performance in simultaneous scenarios. Experimental results on PHOENIX2014T and CSL-Daily demonstrate the superiority of our adaptive policy over existing methods, particularly excelling in situations requiring extremely low latency.} } @inproceedings{nunnari-etal-2024-fabeln:lrec, author = {Nunnari, Fabrizio and Avramidis, Eleftherios and Espa{\~n}a-Bonet, Cristina and Gonz{\'a}lez, Marco and Hennes, Anna and Gebhard, Patrick}, title = {{DGS-Fabeln-1}: A Multi-Angle Parallel Corpus of Fairy Tales between {German} {Sign} {Language} and {German} Text}, pages = {4847--4857}, editor = {Calzolari, Nicoletta and Kan, Min-Yen and Hoste, Veronique and Lenci, Alessandro and Sakti, Sakriani and Xue, Nianwen}, booktitle = {2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation ({LREC-COLING} 2024)}, publisher = {{ELRA Language Resources Association (ELRA) and the International Committee on Computational Linguistics (ICCL)}}, address = {Torino, Italy}, day = {20--25}, month = may, year = {2024}, isbn = {978-2-493814-10-4}, language = {english}, url = {https://aclanthology.org/2024.lrec-main.434}, abstract = {We present the acquisition process and the data of DGS-Fabeln-1, a parallel corpus of German text and videos containing German fairy tales interpreted into the German Sign Language (DGS) by a native DGS signer. The corpus contains 573 segments of videos with a total duration of 1 hour and 32 minutes, corresponding with 1428 written sentences. It is the first corpus of semi-naturally expressed DGS that has been filmed from 7 angles, and one of the few sign language (SL) corpora globally which have been filmed from more than 3 angles and where the listener has been simultaneously filmed. The corpus aims at aiding research at SL linguistics, SL machine translation and affective computing, and is freely available for research purposes at the following address: https://doi.org/10.5281/zenodo.10822097.} } @inproceedings{chen-etal-2024-factorized:lrec, author = {Chen, Zhigang and Zhou, Benjia and Li, Jun and Wan, Jun and Lei, Zhen and Jiang, Ning and Lu, Quan and Zhao, Guoqing}, title = {Factorized Learning Assisted with Large Language Model for Gloss-free Sign Language Translation}, pages = {7071--7081}, editor = {Calzolari, Nicoletta and Kan, Min-Yen and Hoste, Veronique and Lenci, Alessandro and Sakti, Sakriani and Xue, Nianwen}, booktitle = {2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation ({LREC-COLING} 2024)}, publisher = {{ELRA Language Resources Association (ELRA) and the International Committee on Computational Linguistics (ICCL)}}, address = {Torino, Italy}, day = {20--25}, month = may, year = {2024}, isbn = {978-2-493814-10-4}, language = {english}, url = {https://aclanthology.org/2024.lrec-main.620}, abstract = {Previous Sign Language Translation (SLT) methods achieve superior performance by relying on gloss annotations. However, labeling high-quality glosses is a labor-intensive task, which limits the further development of SLT. Although some approaches work towards gloss-free SLT through jointly training the visual encoder and translation network, these efforts still suffer from poor performance and inefficient use of the powerful Large Language Model (LLM). Most seriously, we find that directly introducing LLM into SLT will lead to insufficient learning of visual representations as LLM dominates the learning curve. To address these problems, we propose Factorized Learning assisted with Large Language Model (FLa-LLM) for gloss-free SLT. Concretely, we factorize the training process into two stages. In the visual initialing stage, we employ a lightweight translation model after the visual encoder to pre-train the visual encoder. In the LLM fine-tuning stage, we freeze the acquired knowledge in the visual encoder and integrate it with a pre-trained LLM to inspire the LLM's translation potential. This factorized training strategy proves to be highly effective as evidenced by significant improvements achieved across three SLT datasets which are all conducted under the gloss-free setting.} } @inproceedings{walsh-etal-2024-select:lrec, author = {Walsh, Harry and Saunders, Ben and Bowden, Richard}, title = {Select and Reorder: A Novel Approach for Neural Sign Language Production}, pages = {14531--14542}, editor = {Calzolari, Nicoletta and Kan, Min-Yen and Hoste, Veronique and Lenci, Alessandro and Sakti, Sakriani and Xue, Nianwen}, booktitle = {2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation ({LREC-COLING} 2024)}, publisher = {{ELRA Language Resources Association (ELRA) and the International Committee on Computational Linguistics (ICCL)}}, address = {Torino, Italy}, day = {20--25}, month = may, year = {2024}, isbn = {978-2-493814-10-4}, language = {english}, url = {https://aclanthology.org/2024.lrec-main.1266}, abstract = {Sign languages, often categorised as low-resource languages, face significant challenges in achieving accurate translation due to the scarcity of parallel annotated datasets. This paper introduces Select and Reorder (S{\&}R), a novel approach that addresses data scarcity by breaking down the translation process into two distinct steps: Gloss Selection (GS) and Gloss Reordering (GR). Our method leverages large spoken language models and the substantial lexical overlap between source spoken languages and target sign languages to establish an initial alignment. Both steps make use of Non-AutoRegressive (NAR) decoding for reduced computation and faster inference speeds. Through this disentanglement of tasks, we achieve state-of-the-art BLEU and Rouge scores on the Meine DGS Annotated (mDGS) dataset, demonstrating a substantial BLUE-1 improvement of 37.88{\%} in Text to Gloss (T2G) Translation. This innovative approach paves the way for more effective translation models for sign languages, even in resource-constrained settings.} } @inproceedings{kim-etal-2024-signbleu:lrec, author = {Kim, Jung-Ho and Huerta-Enochian, Mathew and Ko, Changyong and Lee, Du Hui}, title = {{SignBLEU}: Automatic Evaluation of Multi-channel Sign Language Translation}, pages = {14796--14811}, editor = {Calzolari, Nicoletta and Kan, Min-Yen and Hoste, Veronique and Lenci, Alessandro and Sakti, Sakriani and Xue, Nianwen}, booktitle = {2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation ({LREC-COLING} 2024)}, publisher = {{ELRA Language Resources Association (ELRA) and the International Committee on Computational Linguistics (ICCL)}}, address = {Torino, Italy}, day = {20--25}, month = may, year = {2024}, isbn = {978-2-493814-10-4}, language = {english}, url = {https://aclanthology.org/2024.lrec-main.1289}, abstract = {Sign languages are multi-channel languages that communicate information through not just the hands (manual signals) but also facial expressions and upper body movements (non-manual signals). However, since automatic sign language translation is usually performed by generating a single sequence of glosses, researchers eschew non-manual and co-occurring manual signals in favor of a simplified list of manual glosses. This can lead to significant information loss and ambiguity. In this paper, we introduce a new task named multi-channel sign language translation (MCSLT) and present a novel metric, SignBLEU, designed to capture multiple signal channels. We validated SignBLEU on a system-level task using three sign language corpora with varied linguistic structures and transcription methodologies and examined its correlation with human judgment through two segment-level tasks. We found that SignBLEU consistently correlates better with human judgment than competing metrics. To facilitate further MCSLT research, we report benchmark scores for the three sign language corpora and release the source code for SignBLEU at https://github.com/eq4all-projects/SignBLEU.} } @inproceedings{bigeard:22036:sign-lang:lrec, author = {Bigeard, Sam and Schulder, Marc and Kopf, Maria and Hanke, Thomas and Vasilaki, Kyriaki and Vacalopoulou, Anna and Goulas, Theodoros and Dimou, Athanasia-Lida and Fotinea, Stavroula-Evita and Efthimiou, Eleni}, title = {Introducing Sign Languages to a Multilingual Wordnet: Bootstrapping Corpora and Lexical Resources of {Greek} {Sign} {Language} and {German} {Sign} {Language}}, pages = {9--15}, editor = {Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and Hochgesang, Julie A. and Kristoffersen, Jette and Mesch, Johanna and Schulder, Marc}, booktitle = {Proceedings of the {LREC2022} 10th Workshop on the Representation and Processing of Sign Languages: Multilingual Sign Language Resources}, maintitle = {13th International Conference on Language Resources and Evaluation ({LREC} 2022)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Marseille, France}, day = {25}, month = jun, year = {2022}, isbn = {979-10-95546-86-3}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/22036.html}, abstract = {Wordnets have been a popular lexical resource type for many years. Their sense-based representation of lexical items and numerous relation structures have been used for a variety of computational and linguistic applications. The inclusion of different wordnets into multilingual wordnet networks has further extended their use into the realm of cross-lingual research. Wordnets have been released for many spoken languages. Research has also been carried out into the creation of wordnets for several sign languages, but none have yet resulted in publicly available datasets. This article presents our own efforts towards an inclusion of sign languages in a multilingual wordnet, starting with Greek Sign Language (GSL) and German Sign Language (DGS). Based on differences in available language resources between GSL and DGS, we trial two workflows with different coverage priorities. We also explore how synergies between both workflows can be leveraged and how future work on additional sign languages could profit from building on existing sign language wordnet data. The results of our work are made publicly available.} } @inproceedings{saunders:70001:sltat:lrec, author = {Saunders, Ben and Camg{\"o}z, Necati Cihan and Bowden, Richard}, title = {Skeletal Graph Self-Attention: Embedding a Skeleton Inductive Bias into Sign Language Production}, pages = {95--102}, editor = {Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and McDonald, John C. and Shterionov, Dimitar and Wolfe, Rosalee}, booktitle = {Proceedings of the 7th International Workshop on Sign Language Translation and Avatar Technology: The Junction of the Visual and the Textual: Challenges and Perspectives}, maintitle = {13th International Conference on Language Resources and Evaluation ({LREC} 2022)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Marseille, France}, day = {24}, month = jun, year = {2022}, isbn = {979-10-95546-82-5}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/2022.sltat-1.15.html}, abstract = {Recent approaches to Sign Language Production (SLP) have adopted spoken language Neural Machine Translation (NMT) architectures, applied without sign-specific modifications. In addition, these works represent sign language as a sequence of skeleton pose vectors, projected to an abstract representation with no inherent skeletal structure. In this paper, we represent sign language sequences as a skeletal graph structure, with joints as nodes and both spatial and temporal connections as edges. To operate on this graphical structure, we propose Skeletal Graph Self-Attention (SGSA), a novel graphical attention layer that embeds a skeleton inductive bias into the SLP model. Retaining the skeletal feature representation throughout, we directly apply a spatio-temporal adjacency matrix into the self-attention formulation. This provides structure and context to each skeletal joint that is not possible when using a non-graphical abstract representation, enabling fluid and expressive sign language production. We evaluate our Skeletal Graph Self-Attention architecture on the challenging RWTH-PHOENIX-Weather-2014T (PHOENIX14T) dataset, achieving state-of-the-art back translation performance with an 8{\%} and 7{\%} improvement over competing methods for the dev and test sets.} } @inproceedings{walsh:70007:sltat:lrec, author = {Walsh, Harry and Saunders, Ben and Bowden, Richard}, title = {Changing the Representation: Examining Language Representation for Neural Sign Language Production}, pages = {117--124}, editor = {Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and McDonald, John C. and Shterionov, Dimitar and Wolfe, Rosalee}, booktitle = {Proceedings of the 7th International Workshop on Sign Language Translation and Avatar Technology: The Junction of the Visual and the Textual: Challenges and Perspectives}, maintitle = {13th International Conference on Language Resources and Evaluation ({LREC} 2022)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Marseille, France}, day = {24}, month = jun, year = {2022}, isbn = {979-10-95546-82-5}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/2022.sltat-1.18.html}, abstract = {Neural Sign Language Production (SLP) aims to automatically translate from spoken language sentences to sign language videos. Historically the SLP task has been broken into two steps; Firstly, translating from a spoken language sentence to a gloss sequence and secondly, producing a sign language video given a sequence of glosses. In this paper we apply Natural Language Processing techniques to the first step of the SLP pipeline. We use language models such as BERT and Word2Vec to create better sentence level embeddings, and apply several tokenization techniques, demonstrating how these improve performance on the low resource translation task of Text to Gloss. We introduce Text to HamNoSys (T2H) translation, and show the advantages of using a phonetic representation for sign language translation rather than a sign level gloss representation. Furthermore, we use HamNoSys to extract the hand shape of a sign and use this as additional supervision during training, further increasing the performance on T2H. Assembling best practise, we achieve a BLEU-4 score of 26.99 on the MineDGS dataset and 25.09 on PHOENIX14T, two new state-of-the-art baselines.} } @inproceedings{sisto-etal-2022-challenges:lrec, author = {De Sisto, Mirella and Vandeghinste, Vincent and Egea G{\'o}mez, Santiago and De Coster, Mathieu and Shterionov, Dimitar}, title = {Challenges with Sign Language Datasets for Sign Language Recognition and Translation}, pages = {2478--2487}, editor = {Calzolari, Nicoletta and B{\'e}chet, Fr{\'e}d{\'e}ric and Blache, Philippe and Choukri, Khalid and Cieri, Christopher and Declerck, Thierry and Goggi, Sara and Isahara, Hitoshi and Maegaard, Bente and Mariani, Joseph and Mazo, H{\'e}l{\`e}ne and Odijk, Jan and Piperidis, Stelios}, booktitle = {13th International Conference on Language Resources and Evaluation ({LREC} 2022)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Marseille, France}, day = {20--25}, month = jun, year = {2022}, isbn = {979-10-95546-72-6}, language = {english}, url = {https://aclanthology.org/2022.lrec-1.264}, abstract = {Sign Languages (SLs) are the primary means of communication for at least half a million people in Europe alone. However, the development of SL recognition and translation tools is slowed down by a series of obstacles concerning resource scarcity and standardization issues in the available data. The former challenge relates to the volume of data available for machine learning as well as the time required to collect and process new data. The latter obstacle is linked to the variety of the data, i.e., annotation formats are not unified and vary amongst different resources. The available data formats are often not suitable for machine learning, obstructing the provision of automatic tools based on neural models. In the present paper, we give an overview of these challenges by comparing various SL corpora and SL machine learning datasets. Furthermore, we propose a framework to address the lack of standardization at format level, unify the available resources and facilitate SL research for different languages. Our framework takes ELAN files as inputs and returns textual and visual data ready to train SL recognition and translation models. We present a proof of concept, training neural translation models on the data produced by the proposed framework.} } @inproceedings{tamer:20032:sign-lang:lrec, author = {Tamer, Nazif Can and Sara{\c c}lar, Murat}, title = {Cross-Lingual Keyword Search for Sign Language}, pages = {217--223}, editor = {Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and Hochgesang, Julie A. and Kristoffersen, Jette and Mesch, Johanna}, booktitle = {Proceedings of the {LREC2020} 9th Workshop on the Representation and Processing of Sign Languages: Sign Language Resources in the Service of the Language Community, Technological Challenges and Application Perspectives}, maintitle = {12th International Conference on Language Resources and Evaluation ({LREC} 2020)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Marseille, France}, day = {16}, month = may, year = {2020}, isbn = {979-10-95546-54-2}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/20032.html}, abstract = {Sign language research most often relies on exhaustively annotated and segmented data, which is scarce even for the most studied sign languages. However, parallel corpora consisting of sign language interpreting are rarely explored. By utilizing such data for the task of keyword search, this work aims to enable information retrieval from sign language with the queries from the translated written language. With the written language translations as labels, we train a weakly supervised keyword search model for sign language and further improve the retrieval performance with two context modeling strategies. In our experiments, we compare the gloss retrieval and cross language retrieval performance on RWTH-PHOENIX-Weather 2014T dataset.} } @inproceedings{forster-etal-2014-extensions:lrec, author = {Forster, Jens and Schmidt, Christoph and Koller, Oscar and Bellgardt, Martin and Ney, Hermann}, title = {Extensions of the Sign Language Recognition and Translation Corpus {RWTH}-{PHOENIX}-{Weather}}, pages = {1911--1916}, editor = {Calzolari, Nicoletta and Choukri, Khalid and Declerck, Thierry and Loftsson, Hrafn and Maegaard, Bente and Mariani, Joseph and Moreno, Asuncion and Odijk, Jan and Piperidis, Stelios,}, booktitle = {9th International Conference on Language Resources and Evaluation ({LREC} 2014)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Reykjavik, Iceland}, day = {26--31}, month = may, year = {2014}, isbn = {978-2-9517408-8-4}, language = {english}, url = {https://aclanthology.org/L14-1472}, abstract = {This paper introduces the RWTH-PHOENIX-Weather 2014, a video-based, large vocabulary, German sign language corpus which has been extended over the last two years, tripling the size of the original corpus. The corpus contains weather forecasts simultaneously interpreted into sign language which were recorded from German public TV and manually annotated using glosses on the sentence level and semi-automatically transcribed spoken German extracted from the videos using the open-source speech recognition system RASR. Spatial annotations of the signers' hands as well as shape and orientation annotations of the dominant hand have been added for more than 40k respectively 10k video frames creating one of the largest corpora allowing for quantitative evaluation of object tracking algorithms. Further, over 2k signs have been annotated using the SignWriting annotation system, focusing on the shape, orientation, movement as well as spatial contacts of both hands. Finally, extended recognition and translation setups are defined, and baseline results are presented.} } @inproceedings{kristoffersen:12032:sign-lang:lrec, author = {Kristoffersen, Jette and Troelsg{\aa}rd, Thomas}, title = {Integrating corpora and dictionaries: problems and perspectives, with particular respect to the treatment of sign language}, pages = {95--100}, editor = {Crasborn, Onno and Efthimiou, Eleni and Fotinea, Stavroula-Evita and Hanke, Thomas and Kristoffersen, Jette and Mesch, Johanna}, booktitle = {Proceedings of the {LREC2012} 5th Workshop on the Representation and Processing of Sign Languages: Interactions between Corpus and Lexicon}, maintitle = {8th International Conference on Language Resources and Evaluation ({LREC} 2012)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Istanbul, Turkey}, day = {27}, month = may, year = {2012}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/12032.html}, abstract = {In this paper, we will discuss different possibilities for integration of corpus data with dictionary data, mainly seen from a lexicographic point of view and in a sign language context. For about 25 years a text corpus has been considered a useful, if not necessary tool for editing dictionaries of written and spoken languages. Corpora are equally useful to sign language lexicographers, but sign language corpora have not become accessible until recent years. Nowadays corpora exist, or are being developed, for several sign languages, and we have the possibility of editing new, truly corpus-based sign language dictionaries, and of developing interfaces that integrate corpus and dictionary data. After a brief look at three existing integrated interfaces, one for German, one for Danish, and one for Danish Sign Language, we point out some of the problems that should be considered when making an integrated interface, and, finally, we briefly outline the future perspectives of integrated sign language corpus-dictionary interfaces.} } @inproceedings{forster-etal-2012-rwth:lrec, author = {Forster, Jens and Schmidt, Christoph and Hoyoux, Thomas and Koller, Oscar and Zelle, Uwe and Piater, Justus and Ney, Hermann}, title = {{RWTH}-{PHOENIX}-{Weather}: A Large Vocabulary Sign Language Recognition and Translation Corpus}, pages = {3785--3789}, editor = {Calzolari, Nicoletta and Choukri, Khalid and Declerck, Thierry and Do{\u g}an, Mehmet U{\u g}ur and Maegaard, Bente and Mariani, Joseph and Moreno, Asuncion and Odijk, Jan and Piperidis, Stelios}, booktitle = {8th International Conference on Language Resources and Evaluation ({LREC} 2012)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Istanbul, Turkey}, day = {21--27}, month = may, year = {2012}, isbn = {978-2-9517408-7-7}, language = {english}, url = {https://aclanthology.org/L12-1503}, abstract = {This paper introduces the RWTH-PHOENIX-Weather corpus, a video-based, large vocabulary corpus of German Sign Language suitable for statistical sign language recognition and translation. In contrastto most available sign language data collections, the RWTH-PHOENIX-Weather corpus has not been recorded for linguistic research but for the use in statistical pattern recognition. The corpus contains weather forecasts recorded from German public TV which are manually annotated using glosses distinguishing sign variants, and time boundaries have been marked on the sentence and the gloss level. Further, the spoken German weather forecast has been transcribed in a semi-automatic fashion using a state-of-the-art automatic speech recognition system. Moreover, an additional translation of the glosses into spoken German has been created to capture allowable translation variability. In addition to the corpus, experimental baseline results for hand and head tracking, statistical sign language recognition and translation are presented.} } @inproceedings{zwitserlood:08027:sign-lang:lrec, author = {Zwitserlood, Inge and {\"O}zy{\"u}rek, Asli and Perniss, Pamela}, title = {Annotation of Sign and Gesture Cross-linguistically}, pages = {185--190}, editor = {Crasborn, Onno and Efthimiou, Eleni and Hanke, Thomas and Thoutenhoofd, Ernst D. and Zwitserlood, Inge}, booktitle = {Proceedings of the {LREC2008} 3rd Workshop on the Representation and Processing of Sign Languages: Construction and Exploitation of Sign Language Corpora}, maintitle = {6th International Conference on Language Resources and Evaluation ({LREC} 2008)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Marrakech, Morocco}, day = {1}, month = jun, year = {2008}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/08027.html}, abstract = {In a 5-year project, we compare expressions in the spatial domain between two sign languages (German Sign Language and Turkish Sign Language), the co-speech gestures accompanying two spoken languages (German and Turkish), and the pantomime-like structures used by hearing people (German and Turkish) asked to convey information without speaking. The aim is to discover the similarities and differences in the use of space in expressing referent location and motion between the sign languages, and between the signing, co-speech gesture and no-speech pantomime modes. To this end, we are building a large video corpus of task-related discourse data (about 90 minutes per 15 participants per condition). The data will be described using the IMDI metadata standard and linguistically annotated using ELAN. Parts of the data will be made accessible for research and educational purposes on the Browsable Corpus based at the MPI for Psycholinguistics. \par In this presentation, we report the annotation conventions we have been developing based on collected data. There are two levels of annotation: (i) a descriptive level where we gloss signs and gestures according to the movements/positions of the hands, head, face, and body; and (ii) an analytic/coding level where each sign or gesture is analyzed with respect to the function of establishing and/or maintaining reference in discourse (e.g. through the use of pronouns, classifier predicates, modified verbs, and role shift in signing, and similar forms in gestures). Our conventions combine aspects from other annotation and coding systems developed for sign and gesture (e.g. the ECHO, Corpus NGT, and Auslan Corpus conventions; HamNoSys; gesture coding conventions as developed by Kita, Van Gijn and Van der Hulst), but go beyond them in placing special emphasis on coding both systems with the same parameters. \par On the descriptive level, we developed a 3-dimensional scheme to identify for hand orientation, location, and direction of signs and gestures, allowing comparison across languages. On the analytic/coding level, we devised ways of categorizing how the various spatial expressions in sign and gesture map onto different coreference devices in discourse. \par Parts of the sign and gesture data have now been annotated. We will present some generalizations and conclusions drawn from using our annotation conventions regarding cross-linguistic and sign-gesture comparison. Furthermore, based on our annotation experiences, we will discuss the advantages as well as the shortcomings of our annotation scheme and suggest specific improvements, which the linguistic community needs to consider in terms of ways they can be implemented in the technology of annotation software (such as ELAN).} } @inproceedings{bungeroth-etal-2008-atis:lrec, author = {Bungeroth, Jan and Stein, Daniel and Dreuw, Philippe and Ney, Hermann and Morrissey, Sara and Way, Andy and van Zijl, Lynette}, title = {The {ATIS} Sign Language Corpus}, pages = {2943--2946}, editor = {Calzolari, Nicoletta and Choukri, Khalid and Maegaard, Bente and Mariani, Joseph and Odijk, Jan and Piperidis, Stelios and Tapias, Daniel}, booktitle = {6th International Conference on Language Resources and Evaluation ({LREC} 2008)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Marrakech, Morocco}, day = {26}, month = may, year = {2008}, isbn = {978-2-9517408-4-6}, language = {english}, url = {https://aclanthology.org/L08-1470}, abstract = {Systems that automatically process sign language rely on appropriate data. We therefore present the ATIS sign language corpus that is based on the domain of air travel information. It is available for five languages, English, German, Irish sign language, German sign language and South African sign language. The corpus can be used for different tasks like automatic statistical translation and automatic sign language recognition and it allows the specific modeling of spatial references in signing space.} } @inproceedings{pleissner:06015:sign-lang:lrec, author = {Pleissner, Sandy}, title = {Translation of Natural Speech into Sign Language Based on Semantic Relations}, pages = {74--77}, editor = {Vettori, Chiara}, booktitle = {Proceedings of the {LREC2006} 2nd Workshop on the Representation and Processing of Sign Languages: Lexicographic Matters and Didactic Scenarios}, maintitle = {5th International Conference on Language Resources and Evaluation ({LREC} 2006)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Genoa, Italy}, day = {28}, month = may, year = {2006}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/06015.html}, abstract = {Based on speech observations in children and categories of semantic classes we designed a system which identifies these in natural language and translates them into a sign language. To accomplish this translation, we use algorithms to annotate a set of semantic relations in children's language and hope to regain these sentences from natural source sentences. We define a set of rules used to change the word sequence of origin sentences at every marked relation.} } @inproceedings{bungeroth:04021:sign-lang:lrec, author = {Bungeroth, Jan and Ney, Hermann}, title = {Statistical Sign Language Translation}, pages = {105--108}, editor = {Streiter, Oliver and Vettori, Chiara}, booktitle = {Proceedings of the {LREC2004} Workshop on the Representation and Processing of Sign Languages: From {SignWriting} to Image Processing. Information techniques and their implications for teaching, documentation and communication}, maintitle = {4th International Conference on Language Resources and Evaluation ({LREC} 2004)}, publisher = {{European Language Resources Association (ELRA)}}, address = {Lisbon, Portugal}, day = {30}, month = may, year = {2004}, language = {english}, url = {https://www.sign-lang.uni-hamburg.de/lrec/pub/04021.html}, abstract = {In the field of machine translation, significant progress has been made by using statistical methods. In this paper we suggest a statistical machine translation system for Sign Language and written language, especially for the language pair German Sign Language (DGS) and German. After introducing the system's architecture, statistical machine translation in general and notation systems for Sign Language, the corpus processing is scetched. Finally, preliminary translation results are presented.} }