/*************************************************************************//* *//* Language Technologies Institute *//* Carnegie Mellon University *//* Copyright (c) 1999 *//* All Rights Reserved. *//* *//* Permission is hereby granted, free of charge, to use and distribute *//* this software and its documentation without restriction, including *//* without limitation the rights to use, copy, modify, merge, publish, *//* distribute, sublicense, and/or sell copies of this work, and to *//* permit persons to whom this work is furnished to do so, subject to *//* the following conditions: *//* 1. The code must retain the above copyright notice, this list of *//* conditions and the following disclaimer. *//* 2. Any modifications must be clearly marked as such. *//* 3. Original authors' names are not deleted. *//* 4. The authors' names are not used to endorse or promote products *//* derived from this software without specific prior written *//* permission. *//* *//* CARNEGIE MELLON UNIVERSITY AND THE CONTRIBUTORS TO THIS WORK *//* DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING *//* ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT *//* SHALL CARNEGIE MELLON UNIVERSITY NOR THE CONTRIBUTORS BE LIABLE *//* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES *//* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN *//* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, *//* ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF *//* THIS SOFTWARE. *//* *//*************************************************************************//* Author: Alan W Black (awb@cs.cmu.edu) *//* Date: August 1999 *//*************************************************************************//* *//* test for tokenizer of strings and files *//* *//*************************************************************************/#include <stdio.h>#include "cst_tokenstream.h"int main(int argc, char **argv){ /* need that cool argument parser */ cst_tokenstream *fd; const char *token; fd = ts_open("data.one", cst_ts_default_whitespacesymbols, cst_ts_default_singlecharsymbols, cst_ts_default_prepunctuationsymbols, cst_ts_default_postpunctuationsymbols); while (!ts_eof(fd)) { token = ts_get(fd); printf("ws >%s<\n",fd->whitespace); printf("pp >%s<\n",fd->prepunctuation); printf("tk >%s<\n",token); printf("pp >%s<\n",fd->postpunctuation); printf("fp >%d<\n",fd->file_pos); printf("ln >%d<\n",fd->line_number); printf("\n"); } ts_close(fd); return 0;}
/*************************************************************************/
/* */
/* Language Technologies Institute */
/* Carnegie Mellon University */
/* Copyright (c) 1999 */
/* All Rights Reserved. */
/* */
/* Permission is hereby granted, free of charge, to use and distribute */
/* this software and its documentation without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of this work, and to */
/* permit persons to whom this work is furnished to do so, subject to */
/* the following conditions: */
/* 1. The code must retain the above copyright notice, this list of */
/* conditions and the following disclaimer. */
/* 2. Any modifications must be clearly marked as such. */
/* 3. Original authors' names are not deleted. */
/* 4. The authors' names are not used to endorse or promote products */
/* derived from this software without specific prior written */
/* permission. */
/* */
/* CARNEGIE MELLON UNIVERSITY AND THE CONTRIBUTORS TO THIS WORK */
/* DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING */
/* ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT */
/* SHALL CARNEGIE MELLON UNIVERSITY NOR THE CONTRIBUTORS BE LIABLE */
/* FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES */
/* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN */
/* AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, */
/* ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF */
/* THIS SOFTWARE. */
/* */
/*************************************************************************/
/* Author: Alan W Black (awb@cs.cmu.edu) */
/* Date: August 1999 */
/*************************************************************************/
/* */
/* test for tokenizer of strings and files */
/* */
/*************************************************************************/
#include <stdio.h>
#include "cst_tokenstream.h"
int main(int argc, char **argv)
{
/* need that cool argument parser */
cst_tokenstream *fd;
const char *token;
fd = ts_open("data.one",
cst_ts_default_whitespacesymbols,
cst_ts_default_singlecharsymbols,
cst_ts_default_prepunctuationsymbols,
cst_ts_default_postpunctuationsymbols);
while (!ts_eof(fd))
{
token = ts_get(fd);
printf("ws >%s<\n",fd->whitespace);
printf("pp >%s<\n",fd->prepunctuation);
printf("tk >%s<\n",token);
printf("pp >%s<\n",fd->postpunctuation);
printf("fp >%d<\n",fd->file_pos);
printf("ln >%d<\n",fd->line_number);
printf("\n");
}
ts_close(fd);
return 0;
}