Groonga 3.0.9 Source Code Document
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
io.h
Go to the documentation of this file.
1 /* -*- c-basic-offset: 2 -*- */
2 /* Copyright(C) 2009-2012 Brazil
3 
4  This library is free software; you can redistribute it and/or
5  modify it under the terms of the GNU Lesser General Public
6  License version 2.1 as published by the Free Software Foundation.
7 
8  This library is distributed in the hope that it will be useful,
9  but WITHOUT ANY WARRANTY; without even the implied warranty of
10  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11  Lesser General Public License for more details.
12 
13  You should have received a copy of the GNU Lesser General Public
14  License along with this library; if not, write to the Free Software
15  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 */
17 #ifndef GRN_IO_H
18 #define GRN_IO_H
19 
20 #ifndef GROONGA_IN_H
21 #include "groonga_in.h"
22 #endif /* GROONGA_IN_H */
23 
24 #ifndef GRN_ERROR_H
25 #include "error.h"
26 #endif /* GRN_ERROR_H */
27 
28 #ifdef __cplusplus
29 extern "C" {
30 #endif
31 
32 #ifdef WIN32
33 #ifdef WIN32_FMO_EACH
34 #define GRN_IO_FILE_SIZE 1073741824UL
35 #else /* FMO_EACH */
36 #define GRN_IO_FILE_SIZE 134217728L
37 #endif /* FMO_EACH */
38 #define GRN_IO_COPY grn_io_rdonly
39 #define GRN_IO_UPDATE grn_io_wronly
40 #else /* WIN32 */
41 #define GRN_IO_FILE_SIZE 1073741824UL
42 #define GRN_IO_COPY grn_io_rdwr
43 #define GRN_IO_UPDATE grn_io_rdwr
44 #endif /* WIN32 */
45 
46 typedef enum {
47  grn_io_rdonly,
48  grn_io_wronly,
49  grn_io_rdwr
50 } grn_io_rw_mode;
51 
52 typedef enum {
53  grn_io_auto,
54  grn_io_manual
55 } grn_io_mode;
56 
57 /**** grn_io ****/
58 
59 typedef struct _grn_io grn_io;
60 
61 typedef struct {
62  grn_io *io;
63  grn_ctx *ctx;
64  uint8_t mode;
65  uint8_t tiny_p;
66  uint32_t pseg;
67  uint32_t segment;
68  uint32_t offset;
69  uint32_t size;
70  uint32_t nseg;
71  off_t pos;
72  void *addr;
73  uint32_t diff;
74  int32_t cached;
75 #if defined(WIN32) && defined(WIN32_FMO_EACH)
76  HANDLE fmo;
77 #endif /* defined(WIN32) && defined(WIN32_FMO_EACH) */
78 } grn_io_win;
79 
80 typedef struct {
81  void *map;
82  uint32_t nref;
83  uint32_t count;
84 #if defined(WIN32) && defined(WIN32_FMO_EACH)
85  HANDLE fmo;
86 #endif /* defined(WIN32) && defined(WIN32_FMO_EACH) */
87 } grn_io_mapinfo;
88 
89 typedef struct _grn_io_array_info grn_io_array_info;
90 
91 struct _grn_io_header {
92  char idstr[16];
93  uint32_t type;
94  uint32_t version;
95  uint32_t flags;
96  uint32_t header_size;
97  uint32_t segment_size;
98  uint32_t max_segment;
99  uint32_t n_arrays;
100  uint32_t lock;
101  uint64_t curr_size;
102  uint32_t segment_tail;
103  uint32_t lastmod;
104 };
105 
106 struct _grn_io {
107  char path[PATH_MAX];
108  struct _grn_io_header *header;
109  byte *user_header;
110  grn_io_mapinfo *maps;
111  uint32_t base;
112  uint32_t base_seg;
113  grn_io_mode mode;
114  struct _grn_io_fileinfo *fis;
115  grn_io_array_info *ainfo;
116  uint32_t max_map_seg;
117  uint32_t nmaps;
118  uint32_t nref;
119  uint32_t count;
120  uint8_t flags;
121  uint32_t *lock;
122 };
123 
124 GRN_API grn_io *grn_io_create(grn_ctx *ctx, const char *path,
125  uint32_t header_size, uint32_t segment_size,
126  uint32_t max_segment, grn_io_mode mode,
127  unsigned int flags);
128 grn_io *grn_io_open(grn_ctx *ctx, const char *path, grn_io_mode mode);
129 GRN_API grn_rc grn_io_close(grn_ctx *ctx, grn_io *io);
130 grn_rc grn_io_remove(grn_ctx *ctx, const char *path);
131 grn_rc grn_io_size(grn_ctx *ctx, grn_io *io, uint64_t *size);
132 grn_rc grn_io_rename(grn_ctx *ctx, const char *old_name, const char *new_name);
133 GRN_API void *grn_io_header(grn_io *io);
134 
135 void *grn_io_win_map(grn_io *io, grn_ctx *ctx, grn_io_win *iw, uint32_t segment,
136  uint32_t offset, uint32_t size, grn_io_rw_mode mode);
137 grn_rc grn_io_win_mapv(grn_io_win **list, grn_ctx *ctx, int nent);
138 grn_rc grn_io_win_unmap(grn_io_win *iw);
139 
140 void *grn_io_win_map2(grn_io *io, grn_ctx *ctx, grn_io_win *iw, uint32_t segment,
141  uint32_t offset, uint32_t size, grn_io_rw_mode mode);
142 grn_rc grn_io_win_unmap2(grn_io_win *iw);
143 
144 typedef struct _grn_io_ja_einfo grn_io_ja_einfo;
145 typedef struct _grn_io_ja_ehead grn_io_ja_ehead;
146 
147 struct _grn_io_ja_einfo {
148  uint32_t pos;
149  uint32_t size;
150 };
151 
152 struct _grn_io_ja_ehead {
153  uint32_t size;
154  uint32_t key;
155 };
156 
157 grn_rc grn_io_read_ja(grn_io *io, grn_ctx *ctx, grn_io_ja_einfo *einfo, uint32_t epos,
158  uint32_t key, uint32_t segment, uint32_t offset,
159  void **value, uint32_t *value_len);
160 grn_rc grn_io_write_ja(grn_io *io, grn_ctx *ctx,
161  uint32_t key, uint32_t segment, uint32_t offset,
162  void *value, uint32_t value_len);
163 
164 grn_rc grn_io_write_ja_ehead(grn_io *io, grn_ctx *ctx, uint32_t key,
165  uint32_t segment, uint32_t offset, uint32_t value_len);
166 
167 #define GRN_TABLE_ADD (0x01<<6)
168 #define GRN_TABLE_ADDED (0x01<<7)
169 
170 #define GRN_IO_MAX_RETRY (0x10000)
171 #define GRN_IO_MAX_REF (0x80000000)
172 
173 #define GRN_IO_EXPIRE_GTICK (0x01)
174 #define GRN_IO_EXPIRE_SEGMENT (0x02)
175 #define GRN_IO_TEMPORARY (0x04)
176 
177 void grn_io_seg_map_(grn_ctx *ctx, grn_io *io, uint32_t segno, grn_io_mapinfo *info);
178 
179 /* arguments must be validated by caller;
180  * io mustn't be NULL;
181  * segno must be in valid range;
182  * addr must be set NULL;
183  */
184 #define GRN_IO_SEG_REF(io,segno,addr) do {\
185  grn_io_mapinfo *info = &(io)->maps[segno];\
186  uint32_t nref, retry, *pnref = &info->nref;\
187  if (io->flags & GRN_IO_EXPIRE_SEGMENT) {\
188  if (io->flags & GRN_IO_EXPIRE_GTICK) {\
189  for (retry = 0; !info->map || info->count != grn_gtick; retry++) {\
190  GRN_ATOMIC_ADD_EX(pnref, 1, nref);\
191  if (nref) {\
192  GRN_ATOMIC_ADD_EX(pnref, -1, nref);\
193  if (retry >= GRN_IO_MAX_RETRY) {\
194  GRN_LOG(ctx, GRN_LOG_CRIT, "deadlock detected! in GRN_IO_SEG_REF(%p, %u)", io, segno);\
195  break;\
196  }\
197  GRN_FUTEX_WAIT(pnref);\
198  } else {\
199  info->count = grn_gtick;\
200  if (!info->map) {\
201  grn_io_seg_map_(ctx, io, segno, info);\
202  if (!info->map) {\
203  GRN_LOG(ctx, GRN_LOG_CRIT,\
204  "mmap failed! in GRN_IO_SEG_REF(%p, %u): %s",\
205  io, segno, grn_current_error_message());\
206  }\
207  }\
208  GRN_ATOMIC_ADD_EX(pnref, -1, nref);\
209  GRN_FUTEX_WAKE(pnref);\
210  break;\
211  }\
212  }\
213  } else {\
214  for (retry = 0;; retry++) {\
215  GRN_ATOMIC_ADD_EX(pnref, 1, nref);\
216  if (nref >= GRN_IO_MAX_REF) {\
217  GRN_ATOMIC_ADD_EX(pnref, -1, nref);\
218  if (retry >= GRN_IO_MAX_RETRY) {\
219  GRN_LOG(ctx, GRN_LOG_CRIT, "deadlock detected!! in GRN_IO_SEG_REF(%p, %u, %u)", io, segno, nref);\
220  *pnref = 0; /* force reset */ \
221  break;\
222  }\
223  GRN_FUTEX_WAIT(pnref);\
224  continue;\
225  }\
226  if (nref >= 0x40000000) {\
227  ALERT("strange nref value!! in GRN_IO_SEG_REF(%p, %u, %u)", io, segno, nref); \
228  }\
229  if (!info->map) {\
230  if (nref) {\
231  GRN_ATOMIC_ADD_EX(pnref, -1, nref);\
232  if (retry >= GRN_IO_MAX_RETRY) {\
233  GRN_LOG(ctx, GRN_LOG_CRIT, "deadlock detected!!! in GRN_IO_SEG_REF(%p, %u, %u)", io, segno, nref);\
234  break;\
235  }\
236  GRN_FUTEX_WAIT(pnref);\
237  continue;\
238  } else {\
239  grn_io_seg_map_(ctx, io, segno, info);\
240  if (!info->map) {\
241  GRN_ATOMIC_ADD_EX(pnref, -1, nref);\
242  GRN_LOG(ctx, GRN_LOG_CRIT,\
243  "mmap failed!!! in GRN_IO_SEG_REF(%p, %u, %u): %s",\
244  io, segno, nref, grn_current_error_message());\
245  }\
246  \
247  GRN_FUTEX_WAKE(pnref);\
248  }\
249  }\
250  break;\
251  }\
252  info->count = grn_gtick;\
253  }\
254  } else {\
255  for (retry = 0; !info->map; retry++) {\
256  GRN_ATOMIC_ADD_EX(pnref, 1, nref);\
257  if (nref) {\
258  GRN_ATOMIC_ADD_EX(pnref, -1, nref);\
259  if (retry >= GRN_IO_MAX_RETRY) {\
260  GRN_LOG(ctx, GRN_LOG_CRIT, "deadlock detected!!!! in GRN_IO_SEG_REF(%p, %u)", io, segno);\
261  break;\
262  }\
263  GRN_FUTEX_WAIT(pnref);\
264  } else {\
265  if (!info->map) {\
266  grn_io_seg_map_(ctx, io, segno, info);\
267  if (!info->map) {\
268  GRN_LOG(ctx, GRN_LOG_CRIT,\
269  "mmap failed!!!! in GRN_IO_SEG_REF(%p, %u): %s",\
270  io, segno, grn_current_error_message());\
271  }\
272  }\
273  GRN_ATOMIC_ADD_EX(pnref, -1, nref);\
274  GRN_FUTEX_WAKE(pnref);\
275  break;\
276  }\
277  }\
278  info->count = grn_gtick;\
279  }\
280  addr = info->map;\
281 } while (0)
282 
283 #define GRN_IO_SEG_UNREF(io,segno) do {\
284  if (GRN_IO_EXPIRE_SEGMENT ==\
285  (io->flags & (GRN_IO_EXPIRE_GTICK|GRN_IO_EXPIRE_SEGMENT))) {\
286  uint32_t nref, *pnref = &(io)->maps[segno].nref;\
287  GRN_ATOMIC_ADD_EX(pnref, -1, nref);\
288  }\
289 } while (0)
290 
291 uint32_t grn_io_base_seg(grn_io *io);
292 const char *grn_io_path(grn_io *io);
293 
294 typedef struct _grn_io_array_spec grn_io_array_spec;
295 
296 struct _grn_io_array_spec {
297  uint32_t w_of_element;
298  uint32_t max_n_segments;
299 };
300 
301 struct _grn_io_array_info {
302  uint32_t w_of_elm_in_a_segment;
303  uint32_t elm_mask_in_a_segment;
304  uint32_t max_n_segments;
305  uint32_t element_size;
306  uint32_t *segments;
307  void **addrs;
308 };
309 
310 grn_io *grn_io_create_with_array(grn_ctx *ctx, const char *path, uint32_t header_size,
311  uint32_t segment_size, grn_io_mode mode,
312  int n_arrays, grn_io_array_spec *array_specs);
313 
314 void *grn_io_array_at(grn_ctx *ctx, grn_io *io, uint32_t array, off_t offset, int *flags);
315 
316 void grn_io_segment_alloc(grn_ctx *ctx, grn_io *io, grn_io_array_info *ai,
317  uint32_t lseg, int *flags, void **p);
318 
319 GRN_API grn_rc grn_io_lock(grn_ctx *ctx, grn_io *io, int timeout);
320 GRN_API void grn_io_unlock(grn_io *io);
321 void grn_io_clear_lock(grn_io *io);
322 uint32_t grn_io_is_locked(grn_io *io);
323 
324 #define GRN_IO_ARRAY_AT(io,array,offset,flags,res) do {\
325  grn_io_array_info *ainfo = &(io)->ainfo[array];\
326  uint32_t lseg = (offset) >> ainfo->w_of_elm_in_a_segment;\
327  void **p_ = &ainfo->addrs[lseg];\
328  if (!*p_) {\
329  grn_io_segment_alloc(ctx, (io), ainfo, lseg, (flags), p_);\
330  if (!*p_) { (res) = NULL; break; }\
331  }\
332  *((byte **)(&(res))) = (((byte *)*p_) + \
333  (((offset) & ainfo->elm_mask_in_a_segment) * ainfo->element_size));\
334 } while (0)
335 
336 #define GRN_IO_ARRAY_BIT_AT(io,array,offset,res) do {\
337  uint8_t *ptr_;\
338  int flags_ = 0;\
339  GRN_IO_ARRAY_AT((io), (array), ((offset) >> 3) + 1, &flags_, ptr_);\
340  res = ptr_ ? ((*ptr_ >> ((offset) & 7)) & 1) : 0;\
341 } while (0)
342 
343 #define GRN_IO_ARRAY_BIT_ON(io,array,offset) do {\
344  uint8_t *ptr_;\
345  int flags_ = GRN_TABLE_ADD;\
346  GRN_IO_ARRAY_AT((io), (array), ((offset) >> 3) + 1, &flags_, ptr_);\
347  if (ptr_) { *ptr_ |= (1 << ((offset) & 7)); }\
348 } while (0)
349 
350 #define GRN_IO_ARRAY_BIT_OFF(io,array,offset) do {\
351  uint8_t *ptr_;\
352  int flags_ = GRN_TABLE_ADD;\
353  GRN_IO_ARRAY_AT((io), (array), ((offset) >> 3) + 1, &flags_, ptr_);\
354  if (ptr_) { *ptr_ &= ~(1 << ((offset) & 7)); }\
355 } while (0)
356 
357 #define GRN_IO_ARRAY_BIT_FLIP(io,array,offset) do {\
358  uint8_t *ptr_;\
359  int flags_ = GRN_TABLE_ADD;\
360  GRN_IO_ARRAY_AT((io), (array), ((offset) >> 3) + 1, &flags_, ptr_);\
361  if (ptr_) { *ptr_ ^= (1 << ((offset) & 7)); }\
362 } while (0)
363 
364 void *grn_io_anon_map(grn_ctx *ctx, grn_io_mapinfo *mi, size_t length);
365 void grn_io_anon_unmap(grn_ctx *ctx, grn_io_mapinfo *mi, size_t length);
366 uint32_t grn_io_detect_type(grn_ctx *ctx, const char *path);
367 grn_rc grn_io_set_type(grn_io *io, uint32_t type);
368 uint32_t grn_io_get_type(grn_io *io);
369 
370 grn_rc grn_io_init(void);
371 grn_rc grn_io_fin(void);
372 
373 uint32_t grn_io_expire(grn_ctx *ctx, grn_io *io, int count_thresh, uint32_t limit);
374 uint32_t grn_expire(grn_ctx *ctx, int count_thresh, uint32_t limit);
375 
376 /* encode/decode */
377 
378 #define GRN_B_ENC(v,p) do {\
379  uint8_t *_p = (uint8_t *)p; \
380  uint32_t _v = v; \
381  if (_v < 0x8f) { \
382  *_p++ = _v; \
383  } else if (_v < 0x408f) { \
384  _v -= 0x8f; \
385  *_p++ = 0xc0 + (_v >> 8); \
386  *_p++ = _v & 0xff; \
387  } else if (_v < 0x20408f) { \
388  _v -= 0x408f; \
389  *_p++ = 0xa0 + (_v >> 16); \
390  *_p++ = (_v >> 8) & 0xff; \
391  *_p++ = _v & 0xff; \
392  } else if (_v < 0x1020408f) { \
393  _v -= 0x20408f; \
394  *_p++ = 0x90 + (_v >> 24); \
395  *_p++ = (_v >> 16) & 0xff; \
396  *_p++ = (_v >> 8) & 0xff; \
397  *_p++ = _v & 0xff; \
398  } else { \
399  *_p++ = 0x8f; \
400  memcpy(_p, &_v, sizeof(uint32_t));\
401  _p += sizeof(uint32_t); \
402  } \
403  p = _p; \
404 } while (0)
405 
406 #define GRN_B_ENC_SIZE(v) \
407  ((v) < 0x8f ? 1 : ((v) < 0x408f ? 2 : ((v) < 0x20408f ? 3 : ((v) < 0x1020408f ? 4 : 5))))
408 
409 #define GRN_B_DEC(v,p) do { \
410  uint8_t *_p = (uint8_t *)p; \
411  uint32_t _v = *_p++; \
412  switch (_v >> 4) { \
413  case 0x08 : \
414  if (_v == 0x8f) { \
415  memcpy(&_v, _p, sizeof(uint32_t));\
416  _p += sizeof(uint32_t); \
417  } \
418  break; \
419  case 0x09 : \
420  _v = (_v - 0x90) * 0x100 + *_p++; \
421  _v = _v * 0x100 + *_p++; \
422  _v = _v * 0x100 + *_p++ + 0x20408f; \
423  break; \
424  case 0x0a : \
425  case 0x0b : \
426  _v = (_v - 0xa0) * 0x100 + *_p++; \
427  _v = _v * 0x100 + *_p++ + 0x408f; \
428  break; \
429  case 0x0c : \
430  case 0x0d : \
431  case 0x0e : \
432  case 0x0f : \
433  _v = (_v - 0xc0) * 0x100 + *_p++ + 0x8f; \
434  break; \
435  } \
436  v = _v; \
437  p = _p; \
438 } while (0)
439 
440 #define GRN_B_SKIP(p) do { \
441  uint8_t *_p = (uint8_t *)p; \
442  uint32_t _v = *_p++; \
443  switch (_v >> 4) { \
444  case 0x08 : \
445  if (_v == 0x8f) { \
446  _p += sizeof(uint32_t); \
447  } \
448  break; \
449  case 0x09 : \
450  _p += 3; \
451  break; \
452  case 0x0a : \
453  case 0x0b : \
454  _p += 2; \
455  break; \
456  case 0x0c : \
457  case 0x0d : \
458  case 0x0e : \
459  case 0x0f : \
460  _p += 1; \
461  break; \
462  } \
463  p = _p; \
464 } while (0)
465 
466 #define GRN_B_COPY(p2,p1) do { \
467  uint32_t size = 0, _v = *p1++; \
468  *p2++ = _v; \
469  switch (_v >> 4) { \
470  case 0x08 : \
471  size = (_v == 0x8f) ? 4 : 0; \
472  break; \
473  case 0x09 : \
474  size = 3; \
475  break; \
476  case 0x0a : \
477  case 0x0b : \
478  size = 2; \
479  break; \
480  case 0x0c : \
481  case 0x0d : \
482  case 0x0e : \
483  case 0x0f : \
484  size = 1; \
485  break; \
486  } \
487  while (size--) { *p2++ = *p1++; } \
488 } while (0)
489 
490 #ifdef __cplusplus
491 }
492 #endif
493 
494 #endif /* GRN_IO_H */
Generated on Sun Nov 10 2013 09:49:03 for Groonga 3.0.9 Source Code Document by   doxygen 1.8.1.2