tokenizer.h File Reference

#include <stddef.h>
#include <groonga/plugin.h>

Include dependency graph for tokenizer.h:

This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures
struct	_grn_tokenizer_query
struct	_grn_tokenizer_token

Macros
#define	GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8   "\xEF\xBF\xBE"
#define	GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8_LEN   3
#define	GRN_TOKENIZER_TOKEN_CONTINUE   (0)
#define	GRN_TOKENIZER_TOKEN_LAST   (0x01L<<0)
#define	GRN_TOKENIZER_TOKEN_OVERLAP   (0x01L<<1)
#define	GRN_TOKENIZER_TOKEN_UNMATURED   (0x01L<<2)
#define	GRN_TOKENIZER_TOKEN_REACH_END   (0x01L<<3)
#define	GRN_TOKENIZER_CONTINUE   GRN_TOKENIZER_TOKEN_CONTINUE
#define	GRN_TOKENIZER_LAST   GRN_TOKENIZER_TOKEN_LAST

Typedefs
typedef struct _grn_tokenizer_query	grn_tokenizer_query
typedef struct _grn_tokenizer_token	grn_tokenizer_token
typedef unsigned int	grn_tokenizer_status

Functions
int	grn_tokenizer_charlen (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
int	grn_tokenizer_isspace (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
grn_bool	grn_tokenizer_is_tokenized_delimiter (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
GRN_PLUGIN_EXPORT grn_bool	grn_tokenizer_have_tokenized_delimiter (grn_ctx *ctx, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
GRN_PLUGIN_EXPORT grn_tokenizer_query *	grn_tokenizer_query_open (grn_ctx *ctx, int num_args, grn_obj **args, unsigned int normalize_flags)
grn_tokenizer_query *	grn_tokenizer_query_create (grn_ctx *ctx, int num_args, grn_obj **args)
GRN_PLUGIN_EXPORT void	grn_tokenizer_query_close (grn_ctx *ctx, grn_tokenizer_query *query)
void	grn_tokenizer_query_destroy (grn_ctx *ctx, grn_tokenizer_query *query)
GRN_PLUGIN_EXPORT void	grn_tokenizer_token_init (grn_ctx *ctx, grn_tokenizer_token *token)
GRN_PLUGIN_EXPORT void	grn_tokenizer_token_fin (grn_ctx *ctx, grn_tokenizer_token *token)
GRN_PLUGIN_EXPORT void	grn_tokenizer_token_push (grn_ctx *ctx, grn_tokenizer_token *token, const char *str_ptr, unsigned int str_length, grn_tokenizer_status status)
GRN_PLUGIN_EXPORT const char *	grn_tokenizer_tokenized_delimiter_next (grn_ctx *ctx, grn_tokenizer_token *token, const char *str_ptr, unsigned int str_length, grn_encoding encoding)
GRN_PLUGIN_EXPORT grn_rc	grn_tokenizer_register (grn_ctx *ctx, const char *plugin_name_ptr, unsigned int plugin_name_length, grn_proc_func *init, grn_proc_func *next, grn_proc_func *fin)

Macro Definition Documentation

#define GRN_TOKENIZER_CONTINUE   GRN_TOKENIZER_TOKEN_CONTINUE

Definition at line 171 of file tokenizer.h.

#define GRN_TOKENIZER_LAST   GRN_TOKENIZER_TOKEN_LAST

Definition at line 172 of file tokenizer.h.

#define GRN_TOKENIZER_TOKEN_CONTINUE   (0)

Definition at line 155 of file tokenizer.h.

#define GRN_TOKENIZER_TOKEN_LAST   (0x01L<<0)

Definition at line 157 of file tokenizer.h.

#define GRN_TOKENIZER_TOKEN_OVERLAP   (0x01L<<1)

Definition at line 159 of file tokenizer.h.

#define GRN_TOKENIZER_TOKEN_REACH_END   (0x01L<<3)

Definition at line 163 of file tokenizer.h.

#define GRN_TOKENIZER_TOKEN_UNMATURED   (0x01L<<2)

Definition at line 161 of file tokenizer.h.

#define GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8   "\xEF\xBF\xBE"

Definition at line 29 of file tokenizer.h.

#define GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8_LEN   3

Definition at line 30 of file tokenizer.h.

Typedef Documentation

typedef struct _grn_tokenizer_query grn_tokenizer_query

Definition at line 77 of file tokenizer.h.

typedef unsigned int grn_tokenizer_status

Definition at line 152 of file tokenizer.h.

typedef struct _grn_tokenizer_token grn_tokenizer_token

Definition at line 128 of file tokenizer.h.

Function Documentation

int grn_tokenizer_charlen	(	grn_ctx *	ctx,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_encoding	encoding
	)

Definition at line 35 of file tokenizer.c.

Here is the call graph for this function:

GRN_PLUGIN_EXPORT grn_bool grn_tokenizer_have_tokenized_delimiter	(	grn_ctx *	ctx,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_encoding	encoding
	)

Definition at line 71 of file tokenizer.c.

Here is the call graph for this function:

Here is the caller graph for this function:

grn_bool grn_tokenizer_is_tokenized_delimiter	(	grn_ctx *	ctx,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_encoding	encoding
	)

Definition at line 52 of file tokenizer.c.

Here is the caller graph for this function:

int grn_tokenizer_isspace	(	grn_ctx *	ctx,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_encoding	encoding
	)

Definition at line 45 of file tokenizer.c.

Here is the call graph for this function:

GRN_PLUGIN_EXPORT void grn_tokenizer_query_close	(	grn_ctx *	ctx,
		grn_tokenizer_query *	query
	)

Definition at line 202 of file tokenizer.c.

Here is the call graph for this function:

Here is the caller graph for this function:

grn_tokenizer_query* grn_tokenizer_query_create	(	grn_ctx *	ctx,
		int	num_args,
		grn_obj **	args
	)

Definition at line 196 of file tokenizer.c.

Here is the call graph for this function:

void grn_tokenizer_query_destroy	(	grn_ctx *	ctx,
		grn_tokenizer_query *	query
	)

Definition at line 216 of file tokenizer.c.

Here is the call graph for this function:

GRN_PLUGIN_EXPORT grn_tokenizer_query* grn_tokenizer_query_open	(	grn_ctx *	ctx,
		int	num_args,
		grn_obj **	args,
		unsigned int	normalize_flags
	)

Definition at line 100 of file tokenizer.c.

Here is the call graph for this function:

Here is the caller graph for this function:

GRN_PLUGIN_EXPORT grn_rc grn_tokenizer_register	(	grn_ctx *	ctx,
		const char *	plugin_name_ptr,
		unsigned int	plugin_name_length,
		grn_proc_func *	init,
		grn_proc_func *	next,
		grn_proc_func *	fin
	)

Definition at line 285 of file tokenizer.c.

Here is the call graph for this function:

Here is the caller graph for this function:

GRN_PLUGIN_EXPORT void grn_tokenizer_token_fin	(	grn_ctx *	ctx,
		grn_tokenizer_token *	token
	)

Definition at line 229 of file tokenizer.c.

GRN_PLUGIN_EXPORT void grn_tokenizer_token_init	(	grn_ctx *	ctx,
		grn_tokenizer_token *	token
	)

Definition at line 222 of file tokenizer.c.

GRN_PLUGIN_EXPORT void grn_tokenizer_token_push	(	grn_ctx *	ctx,
		grn_tokenizer_token *	token,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_tokenizer_status	status
	)

Definition at line 236 of file tokenizer.c.

Here is the call graph for this function:

Here is the caller graph for this function:

GRN_PLUGIN_EXPORT const char* grn_tokenizer_tokenized_delimiter_next	(	grn_ctx *	ctx,
		grn_tokenizer_token *	token,
		const char *	str_ptr,
		unsigned int	str_length,
		grn_encoding	encoding
	)

Definition at line 247 of file tokenizer.c.

Here is the call graph for this function: