token.c File Reference

#include "groonga_in.h"
#include <string.h>
#include <ctype.h>
#include "ctx_impl.h"
#include "token.h"
#include "pat.h"
#include "dat.h"
#include "hash.h"
#include "string_in.h"
#include "plugin_in.h"
#include <groonga/tokenizer.h>

Include dependency graph for token.c:

Go to the source code of this file.

Data Structures
struct	grn_uvector_tokenizer
struct	grn_delimited_tokenizer
struct	grn_ngram_tokenizer

Macros
#define	DEF_TOKENIZER(name, init, next, fin, vars)

Functions
grn_rc	grn_token_init (void)
grn_rc	grn_token_fin (void)
grn_token *	grn_token_open (grn_ctx *ctx, grn_obj *table, const char *str, size_t str_len, grn_token_mode mode, unsigned int flags)
grn_id	grn_token_next (grn_ctx *ctx, grn_token *token)
grn_rc	grn_token_close (grn_ctx *ctx, grn_token *token)
grn_rc	grn_db_init_mecab_tokenizer (grn_ctx *ctx)
grn_rc	grn_db_init_builtin_tokenizers (grn_ctx *ctx)

Variables
grn_obj *	grn_token_uvector = NULL

Macro Definition Documentation

#define DEF_TOKENIZER	(		name,
			init,
			next,
			fin,
			vars
	)

Value:

(grn_proc_create(ctx, (name), (sizeof(name) - 1),\
                   GRN_PROC_TOKENIZER, (init), (next), (fin), 3, (vars)))

Definition at line 717 of file token.c.

Function Documentation

grn_rc grn_db_init_builtin_tokenizers

(

grn_ctx *

ctx

)

Definition at line 722 of file token.c.

Here is the caller graph for this function:

grn_rc grn_db_init_mecab_tokenizer

(

grn_ctx *

ctx

)

Definition at line 694 of file token.c.

Here is the call graph for this function:

Here is the caller graph for this function:

grn_rc grn_token_close	(	grn_ctx *	ctx,
		grn_token *	token
	)

Definition at line 676 of file token.c.

Here is the call graph for this function:

Here is the caller graph for this function:

grn_rc grn_token_fin

(

void

)

Definition at line 489 of file token.c.

Here is the caller graph for this function:

grn_rc grn_token_init

(

void

)

Definition at line 474 of file token.c.

Here is the caller graph for this function:

grn_id grn_token_next	(	grn_ctx *	ctx,
		grn_token *	token
	)

Definition at line 558 of file token.c.

Here is the call graph for this function:

Here is the caller graph for this function:

grn_token* grn_token_open	(	grn_ctx *	ctx,
		grn_obj *	table,
		const char *	str,
		size_t	str_len,
		grn_token_mode	mode,
		unsigned int	flags
	)

Definition at line 495 of file token.c.

Here is the call graph for this function:

Here is the caller graph for this function:

Variable Documentation

grn_obj* grn_token_uvector = NULL

Definition at line 30 of file token.c.