tokenizer.c File Reference

#include "groonga_in.h"
#include "groonga/tokenizer.h"
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#include "ctx.h"
#include "db.h"
#include "str.h"
#include "string_in.h"
#include "token.h"

Include dependency graph for tokenizer.c:

Go to the source code of this file.

Functions
int	grn_tokenizer_charlen (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
int	grn_tokenizer_isspace (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
grn_bool	grn_tokenizer_is_tokenized_delimiter (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
grn_bool	grn_tokenizer_have_tokenized_delimiter (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
grn_tokenizer_query *	grn_tokenizer_query_open (grn_ctx *ctx, int num_args, grn_obj **args, unsigned int normalize_flags)
grn_tokenizer_query *	grn_tokenizer_query_create (grn_ctx *ctx, int num_args, grn_obj **args)
void	grn_tokenizer_query_close (grn_ctx *ctx, grn_tokenizer_query *query)
void	grn_tokenizer_query_destroy (grn_ctx *ctx, grn_tokenizer_query *query)
void	grn_tokenizer_token_init (grn_ctx *ctx, grn_tokenizer_token *token)
void	grn_tokenizer_token_fin (grn_ctx *ctx, grn_tokenizer_token *token)
void	grn_tokenizer_token_push (grn_ctx *ctx, grn_tokenizer_token *token, const char *str_ptr, unsigned int str_length, grn_tokenizer_status status)
const char *	grn_tokenizer_tokenized_delimiter_next (grn_ctx *ctx, grn_tokenizer_token *token, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
grn_rc	grn_tokenizer_register (grn_ctx *ctx, const char *plugin_name_ptr, unsigned int plugin_name_length, grn_proc_func *init, grn_proc_func *next, grn_proc_func *fin)

Function Documentation

int grn_tokenizer_charlen	(	grn_ctx *	ctx,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_encoding	encoding
	)

Definition at line 35 of file tokenizer.c.

Here is the call graph for this function:

grn_bool grn_tokenizer_have_tokenized_delimiter	(	grn_ctx *	ctx,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_encoding	encoding
	)

Definition at line 71 of file tokenizer.c.

Here is the call graph for this function:

Here is the caller graph for this function:

grn_bool grn_tokenizer_is_tokenized_delimiter	(	grn_ctx *	ctx,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_encoding	encoding
	)

Definition at line 52 of file tokenizer.c.

Here is the caller graph for this function:

int grn_tokenizer_isspace	(	grn_ctx *	ctx,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_encoding	encoding
	)

Definition at line 45 of file tokenizer.c.

Here is the call graph for this function:

void grn_tokenizer_query_close	(	grn_ctx *	ctx,
		grn_tokenizer_query *	query
	)

Definition at line 202 of file tokenizer.c.

Here is the call graph for this function:

Here is the caller graph for this function:

grn_tokenizer_query* grn_tokenizer_query_create	(	grn_ctx *	ctx,
		int	num_args,
		grn_obj **	args
	)

Definition at line 196 of file tokenizer.c.

Here is the call graph for this function:

void grn_tokenizer_query_destroy	(	grn_ctx *	ctx,
		grn_tokenizer_query *	query
	)

Definition at line 216 of file tokenizer.c.

Here is the call graph for this function:

grn_tokenizer_query* grn_tokenizer_query_open	(	grn_ctx *	ctx,
		int	num_args,
		grn_obj **	args,
		unsigned int	normalize_flags
	)

Definition at line 100 of file tokenizer.c.

Here is the call graph for this function:

Here is the caller graph for this function:

grn_rc grn_tokenizer_register	(	grn_ctx *	ctx,
		const char *	plugin_name_ptr,
		unsigned int	plugin_name_length,
		grn_proc_func *	init,
		grn_proc_func *	next,
		grn_proc_func *	fin
	)

Definition at line 285 of file tokenizer.c.

Here is the call graph for this function:

Here is the caller graph for this function:

void grn_tokenizer_token_fin	(	grn_ctx *	ctx,
		grn_tokenizer_token *	token
	)

Definition at line 229 of file tokenizer.c.

void grn_tokenizer_token_init	(	grn_ctx *	ctx,
		grn_tokenizer_token *	token
	)

Definition at line 222 of file tokenizer.c.

void grn_tokenizer_token_push	(	grn_ctx *	ctx,
		grn_tokenizer_token *	token,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_tokenizer_status	status
	)

Definition at line 236 of file tokenizer.c.

Here is the call graph for this function:

Here is the caller graph for this function:

const char* grn_tokenizer_tokenized_delimiter_next	(	grn_ctx *	ctx,
		grn_tokenizer_token *	token,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_encoding	encoding
	)

Definition at line 247 of file tokenizer.c.

Here is the call graph for this function: