From: Magne Mahre Date: April 21 2010 9:56pm Subject: bzr commit into mysql-trunk-runtime branch (magne.mahre:3007) Bug#49177 List-Archive: http://lists.mysql.com/commits/106356 X-Bug: 49177 Message-Id: <0L180023DY9QQKD0@terra-smin.broadpark.no> MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="Boundary_(ID_N+IfgbWxk7Tneokbn+X1yA)" --Boundary_(ID_N+IfgbWxk7Tneokbn+X1yA) MIME-version: 1.0 Content-type: text/plain; CHARSET=US-ASCII Content-transfer-encoding: 7BIT Content-disposition: inline #At file:///data/z/mysql-trunk-runtime-49177/ based on revid:jon.hauglid@stripped 3007 Magne Mahre 2010-04-21 Bug#49177 table_cache scalability problems (Note: MyISAM issue only) As the table cache size is increased, cache lookups and (in particular) misses became increasingly more expensive. The problem was caused by the cache lookup mechanism, which was based on traversing a linked list, of length, comparing the file names. A hash table has been introduced to reduce the lookup time. The linked list and the hash table are used in parallel, with the linked list used for implementing LRU functionality, and for operations that need to iterate through the entire cache. The hash table is only used for lookup. modified: storage/myisam/mi_close.c storage/myisam/mi_open.c storage/myisam/mi_static.c storage/myisam/myisamdef.h === modified file 'storage/myisam/mi_close.c' --- a/storage/myisam/mi_close.c 2009-12-05 01:26:15 +0000 +++ b/storage/myisam/mi_close.c 2010-04-21 21:56:05 +0000 @@ -54,7 +54,11 @@ int mi_close(register MI_INFO *info) info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED); } flag= !--share->reopen; - myisam_open_list=list_delete(myisam_open_list,&info->open_list); + + /* delete from both the hash and the list */ + my_hash_delete(&myisam_open_hash, info); + myisam_open_list= list_delete(myisam_open_list, &info->open_list); + mysql_mutex_unlock(&share->intern_lock); my_free(mi_get_rec_buff_ptr(info, info->rec_buff), MYF(MY_ALLOW_ZERO_PTR)); === modified file 'storage/myisam/mi_open.c' --- a/storage/myisam/mi_open.c 2009-12-05 01:26:15 +0000 +++ b/storage/myisam/mi_open.c 2010-04-21 21:56:05 +0000 @@ -1,4 +1,4 @@ -/* Copyright (C) 2000-2006 MySQL AB, 2008-2009 Sun Microsystems, Inc +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -18,6 +18,7 @@ #include "fulltext.h" #include "sp_defs.h" #include "rt_index.h" +#include #include #if defined(MSDOS) || defined(__WIN__) @@ -44,6 +45,19 @@ if (pos > end_pos) \ } +static const int myisam_open_hash_size= 32; + +/* retrieve the field used as hash key from the MI_INFO record */ +uchar *mi_info_hash_get_key(const uchar *record, size_t *length, + my_bool not_used __attribute__ ((unused))) +{ + MI_INFO *info= (MI_INFO *) record; + *length= strlen(info->filename); + return (uchar *) info->filename; +} + + + /****************************************************************************** ** Return the shared struct if the table is already open. ** In MySQL the server will handle version issues. @@ -51,15 +65,22 @@ if (pos > end_pos) \ MI_INFO *test_if_reopen(char *filename) { - LIST *pos; + HASH_SEARCH_STATE current_record; + int flen= strlen(filename); - for (pos=myisam_open_list ; pos ; pos=pos->next) + for (info= (MI_INFO *) my_hash_first(&myisam_open_hash, + (uchar *) filename, flen, + ¤t_record); + info; + info= (MI_INFO *) my_hash_next(&myisam_open_hash, + (uchar *) filename, flen, + ¤t_record)) { - MI_INFO *info=(MI_INFO*) pos->data; MYISAM_SHARE *share=info->s; - if (!strcmp(share->unique_file_name,filename) && share->last_version) + if (share->last_version) return info; } + return 0; } @@ -104,6 +125,17 @@ MI_INFO *mi_open(const char *name, int m } mysql_mutex_lock(&THR_LOCK_myisam); + + /* create hash table on the first call to mi_open */ + if (my_hash_init_opt(&myisam_open_hash, &my_charset_bin, + myisam_open_hash_size, 0, 0, + mi_info_hash_get_key, 0, 0)) + { + my_errno= ENOMEM; + goto err; + } + + if (!(old_info=test_if_reopen(name_buff))) { share= &share_buff; @@ -655,7 +687,9 @@ MI_INFO *mi_open(const char *name, int m thr_lock_data_init(&share->lock,&m_info->lock,(void*) m_info); #endif m_info->open_list.data=(void*) m_info; - myisam_open_list=list_add(myisam_open_list,&m_info->open_list); + /* Add record to both the list and the hash */ + myisam_open_list= list_add(myisam_open_list,&m_info->open_list); + my_hash_insert(&myisam_open_hash, m_info); mysql_mutex_unlock(&THR_LOCK_myisam); if (myisam_log_file >= 0) === modified file 'storage/myisam/mi_static.c' --- a/storage/myisam/mi_static.c 2010-02-06 16:13:42 +0000 +++ b/storage/myisam/mi_static.c 2010-04-21 21:56:05 +0000 @@ -1,4 +1,4 @@ -/* Copyright (C) 2000-2002, 2004-2005 MySQL AB, 2008-2009 Sun Microsystems, Inc +/* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -22,7 +22,9 @@ #include "myisamdef.h" #endif -LIST *myisam_open_list=0; +LIST *myisam_open_list=0; +HASH myisam_open_hash= {.blength= 0 }; + uchar NEAR myisam_file_magic[]= { (uchar) 254, (uchar) 254,'\007', '\001', }; uchar NEAR myisam_pack_file_magic[]= === modified file 'storage/myisam/myisamdef.h' --- a/storage/myisam/myisamdef.h 2009-12-17 19:16:54 +0000 +++ b/storage/myisam/myisamdef.h 2010-04-21 21:56:05 +0000 @@ -18,6 +18,7 @@ #include "myisam.h" /* Structs & some defines */ #include "myisampack.h" /* packing of keys */ #include +#include #ifdef THREAD #include #include @@ -475,6 +476,7 @@ extern mysql_mutex_t THR_LOCK_myisam; /* Some extern variables */ extern LIST *myisam_open_list; +extern HASH myisam_open_hash; extern uchar NEAR myisam_file_magic[],NEAR myisam_pack_file_magic[]; extern uint NEAR myisam_read_vec[],NEAR myisam_readnext_vec[]; extern uint myisam_quick_table_bits; --Boundary_(ID_N+IfgbWxk7Tneokbn+X1yA) MIME-version: 1.0 Content-type: text/bzr-bundle; CHARSET=US-ASCII; name="bzr/magne.mahre@stripped" Content-transfer-encoding: 7BIT Content-disposition: inline; filename="bzr/magne.mahre@stripped" # Bazaar merge directive format 2 (Bazaar 0.90) # revision_id: magne.mahre@stripped # target_branch: file:///data/z/mysql-trunk-runtime-49177/ # testament_sha1: 8fbdaa7782cd5bb9cc4b1fc3cc0563ac5e7daa65 # timestamp: 2010-04-21 23:56:13 +0200 # base_revision_id: jon.hauglid@stripped\ # lfux3cuqr3387yux # # Begin bundle IyBCYXphYXIgcmV2aXNpb24gYnVuZGxlIHY0CiMKQlpoOTFBWSZTWeyXROIABMB/gHgQCEBZd/// e2eeAL////pgCru7uGd1fKBRQC9g0oERREBQPRhIpopiTKfkTKn+ielTemTRT2qH6EgD0agaHqD9 UDRAmASeqeo0ABoZAaBkAAAAaag0intJqbUyaaZkjQDQAAaDT1DQNA00lNT9CHqnpPUxGT0PVPRN DIZBkAGEaYDmmRkMmCGjCYI00aMQNMmRgACCSIEAmmhoBT0mE9JM0maKPak02iaGgZICkgKUKf1M Kxdv/DYYZbktcXDHAgyCDDaa4CzrmPsHc4pSRHdKwTSEAq+oHtc/gJJEpohsShRxIqAopMWU2rXZ tj55weCKBgiOArb1mp6mCTmsUjAtClpRRsceL8s7upnBXLQZoSFd97XkxEAMAYU+HpL38l6j5uqM 7Uyox9NDmxUFMiDMKgQCQhIQRtATWIgqOxhjKltC8qra6grVAwCYMDia3ieNb1EVZEgRLiH4nq6e DmqxjihJvjmrBE9N2wqOnUGu/IGm8MMEVyZ8jXQKQWZ2C14GEpTQQKgYDS04jPiCye7EoOUiTHKF qhFkb0WDypAXKGMxIs8ICzljbOtEGe9akKoA5Xw9MqokKtANLsNkpuF+bwMRiYgU6i5JluMAD9O1 bNSt6gQaDTWiEApltVTG2i4ScJfJP0Hb2swQTKZ6xCoGVQPXp+j55ytC5Ii7tgBYeBgX32K9/BHU qOAqZ1YQ4SCosYMLXtOE1w9LpSOWBXU52MVmMI4InkYZnAkwmC2uQSxzmMpGjsi4oMJTSDyKvaWk md2+hpUAWffUCnzwCowYuD1XrrMYys0n0nrgww44WsksViOooW8M+YwzIjtkH20DBiGjL2bM7QIb XUDSOAybVTBnElKjTUwJoCSYxBADgZnUhwLJAVnskAOCqNBWiwpFgxHKzqVqwBiBtGRTrUVhE4Hn AxKaRJsHSwMXnG9amSWkYAlRVRgfo2syqKAmqOtl99LdeVxfO9xoM+satGQ8yQ+gFAxbGuZKFZMf 5yKsqurQpH8wJZGCGN4bBkFetr5XO747ZK5i4GGALy2ZEmkooT6xhyjuBXbIkXn7n/SJEzLi8kfx sOpgey957ce1gZYpVrHerSeHSVRoZQzBEKaodBGkr2kDYXkDA+K5GgFZZfOqMLOXA0EETHmktZqx 8HdtJO0rWkwmZlx/bjcVlNqVpQ3HxOYcjjwBbwWO+zdlnFthuBSYzHWpvTYSJj+t7p7DmQ6DEgVR uipwpmQCkGg5ms2mcnmNQLA7jJffiyZZO0uIBPCkzEpK427e6c1XXU5HsIMMVj2mQHgbw3hWbCQb MzcQNmG4tNM8dlDLGN8mg2dNFC4CQxIzzhXONlRrPh2kP4MTEpGsUSIR1GgiTvIijuImZAwxvOhK QYq9soQNY2LrG4ZBAa8tMSyhUqDDVmh6iBEYtPDSTlGVQESxNecZqZQuKCIw1VAWFkSuqosHKxyB hwsgosTnEsHRcKU0ZpUb7eb+Tno2Wsdd9nniy+CwZG4YPsxYrqPQ74nlIImwdrncwoDEAZBLpUfR Hv+caOVM/tUWOMOw9lygCzc/T3r871TExsYMdL+dDFXVsnzRMnBL+/2Rp51oQhZPZmRwvEaED3aE FDxJNCdqMZQGxS256C4LktbzTFvqukYOvUk53DJ0RCaP1KFCJFKSTBCISEpBLslCjJxgmqOVe+1n +QS+BEeoR4nuPQ+R9f6XpTJC8vI8zO68KFppNZUkd+1CZkLVf9j037o1VoQ8UjKNqWZ7jEHR+Oee JSUEJm9HaH35xQJqKkGIW4l4Ptaaxf6z2m77xvQ5mgOREsJFn45dDSfzIGVBfnsJJbj266VnmeOm O9B6sFFa8RizkL5DQYHA9f4m01kzM4mJqZdzbmibA/A5n8FpyTHI16W/C6aMNYtDT2DkVUR2gjSt 1iUiDSyX7ke4zO4oYWJGmFEHDzpObb1BIxHXx7TdZpOBULA3h9JxKxSwNqSoX3/K71GwzW3kS954 o4G3YY7zPVwTsRdmHBqO4PftjFJjF0XY0qo5UOyhKcOxWIVgbi47+B1Dthe+LGR+YacJovdH6GSF KlyH0CkkVSNNF2HZZBLUjY3wTVQczJe08fAytluW8Mu6CdcsHB0zADIGmlj0wUAPUDFWzaVZPV0r IyBlZMvLDdFSHL1E4E0OkW6gWqxB37ld2VmvggZcDcczyNZgYLkbzEVnWQVEkXmVEGn4g+kGjj6S HWS29oSWw3YIUKxrkM73aqjeC7DkrF39x0oUWk9AVElOgl+vIgNoYs4+XbhxS4weDhNkiclA4DJQ YEyTNsAxRzGSJzXYewIIPsMqmZ2Ttw+amXnUPAzEqsQpkdEgjbeHtBe/sM3mHwffzjmCou+BrCm/ n0RANNVSEZ1hUvZD9SB5JbMITbdH+Tf8QY99rj2J2Fb2b8kVArFcQeCEa0jV1TyiUH5sRicSzoTH QR6HkTPn3G87S/acjqdvaSWXFB7jaHy8Eal7feZi6YaT4Tc0rSDDMwMyK38m1GCkju9cnrVx5KZy UNiWxMwMRgfMY0xDFcjqZQXxt6jUWu9pBkDoxyEv6+J1iYe5C3HUsZUBoGLHklXxoY7zV/tk5jZL 5s3iR0RajHl2c9HusPBm0JolO6VBmbNmOz/uBqtulniCRIbJNbkggOBEkNA0yTDIuF450CjpOl0E qT1xAJNUGEIVMqFsSq3zBcCdLHHHSrZDpgHdT+KGF+Bc4drDF52DF+AewL0RPASLpje5MkaYDxOo dlNaFvGRSAP+xieAxrSLEtsFwirlr/vgZcMD0PqcRhMwb0LDULT49RRjg6LgRYyIAofQyu5gqi20 aNfIYsNh5nmH8u0MzoZGsuUiNvrQyGIHmtQKoFuj+xRIpbEv55yJpau14EELotx6zxd2aQYJgNxM MPrkGxqLrbHrKFnmTLVrry3lDYdEBGfFUnjY1DOxYxPW5yOJJdJltVGVZE9X0kEBm6EkI1f1tWgV jN222Fba5IFJGpHlGNzsIquJia8UZmk8kcGZmZm3l5NFto7sIqjxuWCW0MDHySCO7eRpY6vZnc7m 36HRh1OhAoMRaoHNLqsDfLE+jA6PqaiuwYpVx4XQmWAXxoYBERmyITDFYxTd0XhJlikQT3YRIrRc ayBawoWUjAb2niXfRdFjoqKR72oKQrgYRAHMQwko1lcZaUnmWZi14kYMdUwOC+O2JUrLd5I1/Kyt I5HbxOYeYL7th6ue05GDyO89YbgWRvNQRNYbjX3jJIc8xnv5nN4M0I9OBD88wJcR0I33DmhQNQ2k 75n3B0zM07JTVhVyBZWFa1FhnNeDbP/F3JFOFCQ7JdE4gA== --Boundary_(ID_N+IfgbWxk7Tneokbn+X1yA)--