Below is the list of changes that have just been committed into a local
5.0-fulltext repository of svoj. When svoj does a push these changes will
be propagated to the main repository and, within 24 hours after the
push, to the public repository.
For information on how to access the public repository
see http://dev.mysql.com/doc/mysql/en/installing-source-tree.html
ChangeSet
1.2044 05/12/14 12:45:33 svoj@stripped +9 -0
WL#2575 - Fulltext: Parser plugin for FTS
init()/deinit() parser call implementation.
myisam/myisamdef.h
1.84 05/12/14 12:45:29 svoj@stripped +2 -0
Added number of distinct parsers to MYISAM_SHARE.
Added ftparser_param to MI_INFO.
myisam/mi_open.c
1.97 05/12/14 12:45:29 svoj@stripped +15 -1
Renumber ftparser_nr for each key.
myisam/mi_locking.c
1.40 05/12/14 12:45:29 svoj@stripped +2 -1
Call deinitializer for each initialized parser.
myisam/ftdefs.h
1.34 05/12/14 12:45:29 svoj@stripped +7 -3
Prototypes for new functions were added. MYSQL_FTPARSER_PARAM was added to ft_parse and _mi_ft_parse().
myisam/ft_update.c
1.40 05/12/14 12:45:29 svoj@stripped +8 -4
Call fulltext parser init() function, pass MYSQL_FTPARSER_PARAM, returned by
ftparser_call_initializer(), to _mi_ft_parse()
_mi_ft_parse() accepts additional param now - MYSQL_FTPARSER_PARAM and passes it to parser->parse().
myisam/ft_parser.c
1.55 05/12/14 12:45:29 svoj@stripped +70 -11
Added two functions:
ftparser_call_initializer() - calls parser->init() function if specified and parser is not yet
initialized. Returns MYSQL_FTPARSER_PARAM *.
ftparser_call_deinitializer() - calls parser->deinit() function if specified and parser was
initialized. Deinitializes all parsers.
ft_parse() accepts additional param now - MYSQL_FTPARSER_PARM and passes it to parser->parse().
myisam/ft_nlq_search.c
1.41 05/12/14 12:45:29 svoj@stripped +7 -2
Call fulltext parser init() function, pass MYSQL_FTPARSER_PARAM, returned by
ftparser_call_initializer(), to parser->parse().
myisam/ft_boolean_search.c
1.95 05/12/14 12:45:29 svoj@stripped +39 -38
Call fulltext parser init() function, pass MYSQL_FTPARSER_PARAM, returned by
ftparser_call_initializer(), to parser->parse().
include/myisam.h
1.72 05/12/14 12:45:29 svoj@stripped +1 -0
Distinct fulltext parser number added.
# This is a BitKeeper patch. What follows are the unified diffs for the
# set of deltas contained in the patch. The rest of the patch, the part
# that BitKeeper cares about, is below these diffs.
# User: svoj
# Host: svoj-laptop.mysql.com
# Root: /home/svoj/devel/mysql/CNET/mysql-5.0-cnet
--- 1.71/include/myisam.h 2005-12-02 09:58:43 +04:00
+++ 1.72/include/myisam.h 2005-12-14 12:45:29 +04:00
@@ -195,6 +195,7 @@
uint16 maxlength; /* max length of (packed) key (auto) */
uint16 block_size; /* block_size (auto) */
uint32 version; /* For concurrent read/write */
+ uint32 ftparser_nr; /* distinct ftparser number */
HA_KEYSEG *seg,*end;
struct st_mysql_ftparser *parser; /* Fulltext [pre]parser */
--- 1.54/myisam/ft_parser.c 2005-12-02 09:58:43 +04:00
+++ 1.55/myisam/ft_parser.c 2005-12-14 12:45:29 +04:00
@@ -284,22 +284,81 @@
int ft_parse(TREE *wtree, byte *doc, int doclen, my_bool with_alloc,
- struct st_mysql_ftparser *parser)
+ struct st_mysql_ftparser *parser,
+ MYSQL_FTPARSER_PARAM *param)
{
- MYSQL_FTPARSER_PARAM param;
MY_FT_PARSER_PARAM my_param;
DBUG_ENTER("ft_parse");
DBUG_ASSERT(parser);
my_param.wtree= wtree;
my_param.with_alloc= with_alloc;
- param.mysql_parse= ft_parse_internal;
- param.mysql_add_word= ft_add_word;
- param.ftparser_state= 0;
- param.mysql_ftparam= &my_param;
- param.cs= wtree->custom_arg;
- param.doc= doc;
- param.length= doclen;
- param.mode= MYSQL_FTPARSER_SIMPLE_MODE;
- DBUG_RETURN(parser->parse(¶m));
+ param->mysql_parse= ft_parse_internal;
+ param->mysql_add_word= ft_add_word;
+ param->mysql_ftparam= &my_param;
+ param->cs= wtree->custom_arg;
+ param->doc= doc;
+ param->length= doclen;
+ param->mode= MYSQL_FTPARSER_SIMPLE_MODE;
+ DBUG_RETURN(parser->parse(param));
+}
+
+
+MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info, uint keynr)
+{
+ uint32 ftparser_nr;
+ struct st_mysql_ftparser *parser;
+ if (! info->ftparser_param)
+ {
+ info->ftparser_param= (MYSQL_FTPARSER_PARAM *)
+ my_malloc(sizeof(MYSQL_FTPARSER_PARAM) *
+ info->s->ftparsers, MYF(MY_WME));
+ if (! info->ftparser_param)
+ return 0;
+ bzero(info->ftparser_param,
+ sizeof(MYSQL_FTPARSER_PARAM) * info->s->ftparsers);
+ }
+ if (keynr == NO_SUCH_KEY)
+ {
+ ftparser_nr= 0;
+ parser= &ft_default_parser;
+ }
+ else
+ {
+ ftparser_nr= info->s->keyinfo[keynr].ftparser_nr;
+ parser= info->s->keyinfo[keynr].parser;
+ }
+ if (! info->ftparser_param[ftparser_nr].mysql_add_word)
+ {
+ /* Note, that mysql_add_word is used here as a flag:
+ mysql_add_word == 0 - parser is not initialized
+ mysql_add_word != 0 - parser is initialized, or no
+ initialization needed. */
+ info->ftparser_param[ftparser_nr].mysql_add_word= (void *)1;
+ if (parser->init && parser->init(&info->ftparser_param[ftparser_nr]))
+ return 0;
+ }
+ return &info->ftparser_param[ftparser_nr];
+}
+
+
+void ftparser_call_deinitializer(MI_INFO *info)
+{
+ uint i, keys= info->s->state.header.keys;
+ if (! info->ftparser_param)
+ return;
+ for (i= 0; i < keys; i++)
+ {
+ MI_KEYDEF *keyinfo= &info->s->keyinfo[i];
+ MYSQL_FTPARSER_PARAM *ftparser_param=
+ &info->ftparser_param[keyinfo->ftparser_nr];
+ if (keyinfo->flag & HA_FULLTEXT && ftparser_param->mysql_add_word)
+ {
+ if (keyinfo->parser->deinit)
+ keyinfo->parser->deinit(ftparser_param);
+ ftparser_param->mysql_add_word= 0;
+ }
+ }
+ my_free((gptr)info->ftparser_param, MYF(0));
+ info->ftparser_param= 0;
}
--- 1.39/myisam/ft_update.c 2005-09-01 22:22:47 +05:00
+++ 1.40/myisam/ft_update.c 2005-12-14 12:45:29 +04:00
@@ -96,7 +96,8 @@
/* parses a document i.e. calls ft_parse for every keyseg */
uint _mi_ft_parse(TREE *parsed, MI_INFO *info, uint keynr,
- const byte *record, my_bool with_alloc)
+ const byte *record, my_bool with_alloc,
+ MYSQL_FTPARSER_PARAM *param)
{
FT_SEG_ITERATOR ftsi;
struct st_mysql_ftparser *parser;
@@ -109,7 +110,8 @@
while (_mi_ft_segiterator(&ftsi))
{
if (ftsi.pos)
- if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser))
+ if (ft_parse(parsed, (byte *)ftsi.pos, ftsi.len, with_alloc, parser,
+ param))
DBUG_RETURN(1);
}
DBUG_RETURN(0);
@@ -118,10 +120,12 @@
FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record)
{
TREE ptree;
+ MYSQL_FTPARSER_PARAM *param;
DBUG_ENTER("_mi_ft_parserecord");
-
+ if (! (param= ftparser_call_initializer(info, keynr)))
+ DBUG_RETURN(NULL);
bzero((char*) &ptree, sizeof(ptree));
- if (_mi_ft_parse(&ptree, info, keynr, record,0))
+ if (_mi_ft_parse(&ptree, info, keynr, record, 0, param))
DBUG_RETURN(NULL);
DBUG_RETURN(ft_linearize(&ptree));
--- 1.33/myisam/ftdefs.h 2005-09-12 16:41:46 +05:00
+++ 1.34/myisam/ftdefs.h 2005-12-14 12:45:29 +04:00
@@ -119,10 +119,12 @@
uint _mi_ft_segiterator(FT_SEG_ITERATOR *);
void ft_parse_init(TREE *, CHARSET_INFO *);
-int ft_parse(TREE *, byte *, int, my_bool, struct st_mysql_ftparser *parser);
+int ft_parse(TREE *, byte *, int, my_bool, struct st_mysql_ftparser *parser,
+ MYSQL_FTPARSER_PARAM *param);
FT_WORD * ft_linearize(TREE *);
FT_WORD * _mi_ft_parserecord(MI_INFO *, uint, const byte *);
-uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, my_bool);
+uint _mi_ft_parse(TREE *, MI_INFO *, uint, const byte *, my_bool,
+ MYSQL_FTPARSER_PARAM *param);
FT_INFO *ft_init_nlq_search(MI_INFO *, uint, byte *, uint, uint, byte *);
FT_INFO *ft_init_boolean_search(MI_INFO *, uint, byte *, uint, CHARSET_INFO *);
@@ -142,4 +144,6 @@
float ft_boolean_get_relevance(FT_INFO *);
my_off_t ft_boolean_get_docid(FT_INFO *);
void ft_boolean_reinit_search(FT_INFO *);
-
+extern MYSQL_FTPARSER_PARAM *ftparser_call_initializer(MI_INFO *info,
+ uint keynr);
+extern void ftparser_call_deinitializer(MI_INFO *info);
--- 1.39/myisam/mi_locking.c 2005-05-13 14:08:02 +05:00
+++ 1.40/myisam/mi_locking.c 2005-12-14 12:45:29 +04:00
@@ -21,7 +21,7 @@
isamdatabase.
*/
-#include "myisamdef.h"
+#include "ftdefs.h"
/* lock table by F_UNLCK, F_RDLCK or F_WRLCK */
@@ -56,6 +56,7 @@
{
switch (lock_type) {
case F_UNLCK:
+ ftparser_call_deinitializer(info);
if (info->lock_type == F_RDLCK)
count= --share->r_locks;
else
--- 1.96/myisam/mi_open.c 2005-12-02 09:58:43 +04:00
+++ 1.97/myisam/mi_open.c 2005-12-14 12:45:29 +04:00
@@ -417,8 +417,21 @@
pos->flag=0;
pos++;
}
+ /* Renumber keyinfo->ftparser_nr */
+ share->ftparsers= 1;
+ for (i= 0; i < keys; i++)
+ {
+ MI_KEYDEF *keyinfo= &share->keyinfo[i];
+ if (keyinfo->flag & HA_FULLTEXT && ! keyinfo->ftparser_nr)
+ {
+ uint j;
+ keyinfo->ftparser_nr= share->ftparsers++;
+ for (j= i + 1; j < keys; j++)
+ if (keyinfo->parser == share->keyinfo[j].parser)
+ share->keyinfo[j].ftparser_nr= keyinfo->ftparser_nr;
+ }
+ }
}
-
disk_pos_assert(disk_pos + share->base.fields *MI_COLUMNDEF_SIZE, end_pos);
for (i=j=offset=0 ; i < share->base.fields ; i++)
{
@@ -1048,6 +1061,7 @@
keydef->underflow_block_length=keydef->block_length/3;
keydef->version = 0; /* Not saved */
keydef->parser = &ft_default_parser;
+ keydef->ftparser_nr = 0;
return ptr;
}
--- 1.83/myisam/myisamdef.h 2005-10-27 22:04:31 +05:00
+++ 1.84/myisam/myisamdef.h 2005-12-14 12:45:29 +04:00
@@ -189,6 +189,7 @@
ulong state_diff_length;
uint rec_reflength; /* rec_reflength in use now */
uint unique_name_length;
+ uint32 ftparsers; /* Number of distinct ftparsers + 1 */
File kfile; /* Shared keyfile */
File data_file; /* Shared data file */
int mode; /* mode of file on open */
@@ -227,6 +228,7 @@
/* accumulate indexfile changes between write's */
TREE *bulk_insert;
DYNAMIC_ARRAY *ft1_to_ft2; /* used only in ft1->ft2 conversion */
+ MYSQL_FTPARSER_PARAM *ftparser_param; /* share info between init/deinit */
char *filename; /* parameter to open filename */
uchar *buff, /* Temp area for key */
*lastkey,*lastkey2; /* Last used search key */
--- 1.94/myisam/ft_boolean_search.c 2005-12-02 10:43:43 +04:00
+++ 1.95/myisam/ft_boolean_search.c 2005-12-14 12:45:29 +04:00
@@ -257,28 +257,29 @@
static void _ftb_parse_query(FTB *ftb, byte *query, uint len,
struct st_mysql_ftparser *parser)
{
- MYSQL_FTPARSER_PARAM param;
+ MYSQL_FTPARSER_PARAM *param;
MY_FTB_PARAM ftb_param;
DBUG_ENTER("_ftb_parse_query");
DBUG_ASSERT(parser);
if (ftb->state != UNINITIALIZED)
- return;
+ DBUG_VOID_RETURN;
ftb_param.ftb= ftb;
ftb_param.depth= 0;
ftb_param.ftbe= ftb->root;
ftb_param.up_quot= 0;
- param.mysql_parse= ftb_parse_query_internal;
- param.mysql_add_word= ftb_query_add_word;
- param.ftparser_state= 0;
- param.mysql_ftparam= (void *)&ftb_param;
- param.cs= ftb->charset;
- param.doc= query;
- param.length= len;
- param.mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO;
- parser->parse(¶m);
+ if (! (param= ftparser_call_initializer(ftb->info, ftb->keynr)))
+ DBUG_VOID_RETURN;
+ param->mysql_parse= ftb_parse_query_internal;
+ param->mysql_add_word= ftb_query_add_word;
+ param->mysql_ftparam= (void *)&ftb_param;
+ param->cs= ftb->charset;
+ param->doc= query;
+ param->length= len;
+ param->mode= MYSQL_FTPARSER_FULL_BOOLEAN_INFO;
+ parser->parse(param);
DBUG_VOID_RETURN;
}
@@ -593,30 +594,30 @@
1 is returned if phrase found, 0 else.
*/
-static int _ftb_check_phrase(const byte *document, uint len,
- FTB_EXPR *ftbe, CHARSET_INFO *cs,
- struct st_mysql_ftparser *parser)
+static int _ftb_check_phrase(FTB *ftb, const byte *document, uint len,
+ FTB_EXPR *ftbe, struct st_mysql_ftparser *parser)
{
MY_FTB_PHRASE_PARAM ftb_param;
- MYSQL_FTPARSER_PARAM param;
+ MYSQL_FTPARSER_PARAM *param;
DBUG_ENTER("_ftb_check_phrase");
DBUG_ASSERT(parser);
+ if (! (param= ftparser_call_initializer(ftb->info, ftb->keynr)))
+ DBUG_RETURN(0);
ftb_param.phrase= ftbe->phrase;
ftb_param.document= ftbe->document;
- ftb_param.cs= cs;
+ ftb_param.cs= ftb->charset;
ftb_param.phrase_length= list_length(ftbe->phrase);
ftb_param.document_length= 1;
ftb_param.match= 0;
- param.mysql_parse= ftb_check_phrase_internal;
- param.mysql_add_word= ftb_phrase_add_word;
- param.ftparser_state= 0;
- param.mysql_ftparam= (void *)&ftb_param;
- param.cs= cs;
- param.doc= (byte *)document;
- param.length= len;
- param.mode= MYSQL_FTPARSER_WITH_STOPWORDS;
- parser->parse(¶m);
+ param->mysql_parse= ftb_check_phrase_internal;
+ param->mysql_add_word= ftb_phrase_add_word;
+ param->mysql_ftparam= (void *)&ftb_param;
+ param->cs= ftb->charset;
+ param->doc= (byte *)document;
+ param->length= len;
+ param->mode= MYSQL_FTPARSER_WITH_STOPWORDS;
+ parser->parse(param);
DBUG_RETURN(ftb_param.match ? 1 : 0);
}
@@ -660,8 +661,8 @@
{
if (!ftsi.pos)
continue;
- not_found = ! _ftb_check_phrase(ftsi.pos, ftsi.len,
- ftbe, ftb->charset, parser);
+ not_found = ! _ftb_check_phrase(ftb, ftsi.pos, ftsi.len,
+ ftbe, parser);
}
if (not_found) break;
} /* ftbe->quot */
@@ -825,7 +826,7 @@
FT_SEG_ITERATOR ftsi, ftsi2;
my_off_t docid=ftb->info->lastpos;
MY_FTB_FIND_PARAM ftb_param;
- MYSQL_FTPARSER_PARAM param;
+ MYSQL_FTPARSER_PARAM *param;
struct st_mysql_ftparser *parser= ftb->keynr == NO_SUCH_KEY ?
&ft_default_parser :
ftb->info->s->keyinfo[ftb->keynr].parser;
@@ -834,6 +835,8 @@
return -2.0;
if (!ftb->queue.elements)
return 0;
+ if (! (param= ftparser_call_initializer(ftb->info, ftb->keynr)))
+ return 0;
if (ftb->state != INDEX_SEARCH && docid <= ftb->lastpos)
{
@@ -858,20 +861,18 @@
ftb_param.ftb= ftb;
ftb_param.ftsi= &ftsi2;
- param.mysql_parse= ftb_find_relevance_parse;
- param.mysql_add_word= ftb_find_relevance_add_word;
- param.ftparser_state= 0;
- param.mysql_ftparam= (void *)&ftb_param;
- param.cs= ftb->charset;
- param.mode= MYSQL_FTPARSER_SIMPLE_MODE;
while (_mi_ft_segiterator(&ftsi))
{
if (!ftsi.pos)
continue;
-
- param.doc= (byte *)ftsi.pos;
- param.length= ftsi.len;
- parser->parse(¶m);
+ param->mysql_parse= ftb_find_relevance_parse;
+ param->mysql_add_word= ftb_find_relevance_add_word;
+ param->mysql_ftparam= (void *)&ftb_param;
+ param->cs= ftb->charset;
+ param->mode= MYSQL_FTPARSER_SIMPLE_MODE;
+ param->doc= (byte *)ftsi.pos;
+ param->length= ftsi.len;
+ parser->parse(param);
}
ftbe=ftb->root;
if (ftbe->docid[1]==docid && ftbe->cur_weight>0 &&
--- 1.40/myisam/ft_nlq_search.c 2005-12-02 09:58:43 +04:00
+++ 1.41/myisam/ft_nlq_search.c 2005-12-14 12:45:29 +04:00
@@ -210,6 +210,8 @@
FT_DOC *dptr;
FT_INFO *dlist=NULL;
my_off_t saved_lastpos=info->lastpos;
+ struct st_mysql_ftparser *parser;
+ MYSQL_FTPARSER_PARAM *ftparser_param;
DBUG_ENTER("ft_init_nlq_search");
/* black magic ON */
@@ -223,6 +225,9 @@
aio.keynr=keynr;
aio.charset=info->s->keyinfo[keynr].seg->charset;
aio.keybuff=info->lastkey+info->s->base.max_key_length;
+ parser= info->s->keyinfo[keynr].parser;
+ if (! (ftparser_param= ftparser_call_initializer(info, keynr)))
+ goto err;
bzero(&wtree,sizeof(wtree));
@@ -230,7 +235,7 @@
NULL, NULL);
ft_parse_init(&wtree, aio.charset);
- if (ft_parse(&wtree, query, query_len, 0, info->s->keyinfo[keynr].parser))
+ if (ft_parse(&wtree, query, query_len, 0, parser, ftparser_param))
goto err;
if (tree_walk(&wtree, (tree_walk_action)&walk_and_match, &aio,
@@ -250,7 +255,7 @@
if (!(*info->read_record)(info,docid,record))
{
info->update|= HA_STATE_AKTIV;
- _mi_ft_parse(&wtree, info, keynr, record,1);
+ _mi_ft_parse(&wtree, info, keynr, record, 1, ftparser_param);
}
}
delete_queue(&best);
| Thread |
|---|
| • bk commit into 5.0-fulltext tree (svoj:1.2044) | svoj | 14 Dec |
