Mirrors/MariaDB-server
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-12-28 08:10:14 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Improvements for myisamchk

Browse Source 
These changes was done as part of fixing
MDEV-36858 MariaDB MyISAM secondary indexes silently break for
           tables > 10B rows

Changes done in myisamchk:
- Tables that are checked are opened in readonly mode if --force is not
  used.
- *.MYD files will be opened in readonly mode for repair if --quick
  is used.
- Added information about check progress if --verbose is used.
- Output information about repaired/checked rows every 10000 rows instead
  of every 1000 rows. Note that this also affects aria_chk
- Store open file mode in share->index_mode and share->data_mode instead
  of in share->mode.
- Added new option --keys-active= as a simpler version of keys-used.
- Changed output for "myisamchk -dvv" to get nicer output for tables
  with 10 billion rows.
This commit is contained in:
Monty 2025-07-12 15:45:02 +03:00
parent 675dffb669
commit 6a4fe9923d
12 changed files with 135 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
include/my_base.h
View File

@ -50,6 +50,13 @@
#define HA_OPEN_FOR_CREATE 4096U
#define HA_OPEN_FOR_DROP (1U << 13) /* Open part of drop */
#define HA_OPEN_GLOBAL_TMP_TABLE (1U << 14) /* TMP table used by repliction */
/*
This is to signal that the table will not be cached by the caller
and the table should be open in read-only mode if the tool requests
that
*/
#define HA_OPEN_FORCE_MODE (1U << 15) /* Force open mode */
#define HA_OPEN_DATA_READONLY (1U << 16) /* Use readonly for data */
/*
Allow opening even if table is incompatible as this is for ALTER TABLE which

2
include/my_global.h
View File

@ -1027,7 +1027,7 @@ typedef ulong myf; /* Type of MyFlags in my_funcs */
#define YESNO(X) ((X) ? "yes" : "no")
#define MY_HOW_OFTEN_TO_ALARM 2 /* How often we want info on screen */
#define MY_HOW_OFTEN_TO_WRITE 10000 /* How often we want info on screen */
#define MY_HOW_OFTEN_TO_WRITE 100000 /* How often we want info on screen */
#include <my_byteorder.h>

2
include/myisamchk.h
View File

@ -116,7 +116,7 @@ typedef struct st_handler_check_param
int (*fix_record)(struct st_myisam_info *info, uchar *record, int keynum);
mysql_mutex_t print_msg_mutex;
my_bool need_print_msg_lock;
my_bool need_print_msg_lock, status_reporting;
myf malloc_flags;
} HA_CHECK;

16
mysql-test/main/long_unique.result
View File

@ -42,8 +42,8 @@ Ignored NO
MyISAM file: DATADIR/test/t1
Record format: Packed
Character set: latin1_swedish_ci (8)
Data records: 10 Deleted blocks: 0
Recordlength: 12
Data records: 10 Deleted blocks: 0
Recordlength: 12
table description:
Key Start Len Index Type
@ -130,8 +130,8 @@ insert into t1 values(1),(2),(3),(4),(5),(8),(7);
MyISAM file: DATADIR/test/t1
Record format: Packed
Character set: latin1_swedish_ci (8)
Data records: 7 Deleted blocks: 0
Recordlength: 12
Data records: 7 Deleted blocks: 0
Recordlength: 12
table description:
Key Start Len Index Type
@ -365,8 +365,8 @@ t1 0 e 1 e A NULL NULL NULL YES HASH NO
MyISAM file: DATADIR/test/t1
Record format: Packed
Character set: latin1_swedish_ci (8)
Data records: 8 Deleted blocks: 0
Recordlength: 3040
Data records: 8 Deleted blocks: 0
Recordlength: 3040
table description:
Key Start Len Index Type
@ -722,8 +722,8 @@ t1 0 b 4 h A NULL NULL NULL YES HASH NO
MyISAM file: DATADIR/test/t1
Record format: Packed
Character set: latin1_swedish_ci (8)
Data records: 9 Deleted blocks: 0
Recordlength: 5059
Data records: 9 Deleted blocks: 0
Recordlength: 5059
table description:
Key Start Len Index Type

8
mysql-test/main/myisam.result
View File

@ -2402,8 +2402,8 @@ KEY (c2)
MyISAM file: MYSQLD_DATADIR/test/t1
Record format: Packed
Character set: utf8mb3_general_ci (33)
Data records: 0 Deleted blocks: 0
Recordlength: 94
Data records: 0 Deleted blocks: 0
Recordlength: 94
table description:
Key Start Len Index Type
@ -2690,8 +2690,8 @@ KEY (d)
MyISAM file: MYSQLD_DATADIR/test/t1
Record format: Fixed length
Character set: latin1_swedish_ci (8)
Data records: 0 Deleted blocks: 0
Recordlength: 13
Data records: 0 Deleted blocks: 0
Recordlength: 13
table description:
Key Start Len Index Type

4
mysql-test/suite/vcol/r/vcol_keys_myisam.result
View File

@ -244,8 +244,8 @@ a b c
MyISAM file: datadir/test/t1
Record format: Fixed length
Character set: latin1_swedish_ci (8)
Data records: 1001 Deleted blocks: 0
Recordlength: 9
Data records: 1001 Deleted blocks: 0
Recordlength: 9
table description:
Key Start Len Index Type

18
storage/myisam/mi_check.c
View File

@ -469,6 +469,11 @@ int chk_key(HA_CHECK *param, register MI_INFO *info)
if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
&keys, param->key_crc+key,1))
DBUG_RETURN(-1);
if ((param->testflag & T_WRITE_LOOP) && param->verbose)
{
puts(" \r");
fflush(stdout);
}
if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)))
{
if (keys != info->state->records)
@ -565,7 +570,8 @@ do_stat:
puts("");
}
if (param->key_file_blocks != info->state->key_file_length &&
param->keys_in_use != ~(ulonglong) 0)
mi_is_all_keys_active(share->state.key_map, share->base.keys) &&
!full_text_keys)
mi_check_print_warning(param, "Some data are unreferenced in keyfile");
if (found_keys != full_text_keys)
param->record_checksum=old_record_checksum-init_checksum; /* Remove delete links */
@ -870,6 +876,15 @@ static int chk_index(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
goto err;
}
param->record_checksum+=(ha_checksum) record;
if ((param->testflag & T_WRITE_LOOP) && param->verbose &&
(*keys % WRITE_COUNT) == 0)
{
char llbuff[22];
ulonglong records= info->state->records;
printf("%15s (%3.4f%%)\r", llstr(*keys, llbuff),
((double) *keys / (records > *keys ? records : *keys)) *100);
fflush(stdout);
}
}
if (keypos != endpos)
{
@ -3878,6 +3893,7 @@ static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
{
mi_check_print_error(param,
"Internal error: Keys are not in order from sort");
DBUG_ASSERT(0);
return(1);
}
#endif

2
storage/myisam/mi_close.c
View File

@ -82,7 +82,7 @@ int mi_close(register MI_INFO *info)
(the server might want to reopen it, and mi_lock_database() only
writes the state for non-temp ones)
*/
if (share->mode != O_RDONLY &&
if (share->index_mode != O_RDONLY &&
(mi_is_crashed(info) || (share->temporary && !share->deleting)))
mi_state_info_write(share->kfile, &share->state, 1);
/* Decrement open count must be last I/O on this file. */

2
storage/myisam/mi_dynrec.c
View File

@ -83,7 +83,7 @@ my_bool mi_dynmap_file(MI_INFO *info, my_off_t size)
*/
info->s->file_map= (uchar*)
my_mmap(0, (size_t) size,
info->s->mode==O_RDONLY ? PROT_READ :
info->s->index_mode == O_RDONLY ? PROT_READ :
PROT_READ | PROT_WRITE,
MAP_SHARED | MAP_NORESERVE,
info->dfile, 0L);

38
storage/myisam/mi_open.c
View File

@ -82,7 +82,8 @@ MI_INFO *test_if_reopen(char *filename)
MI_INFO *mi_open(const char *name, int mode, uint open_flags)
{
int lock_error,kfile,open_mode,save_errno,have_rtree=0, realpath_err;
int lock_error,kfile,save_errno,have_rtree=0, realpath_err;
int open_mode, try_open_mode;
uint i,j,len,errpos,head_length,base_pos,offset,info_length,keys,
key_parts,unique_key_parts,base_key_parts,fulltext_keys,uniques;
uint internal_table= open_flags & HA_OPEN_INTERNAL_TABLE;
@ -138,18 +139,31 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
});
DEBUG_SYNC_C("mi_open_kfile");
/*
We first try to open the file on read-write mode to ensure that
the table is usable for future read and write queries in
MariaDB. Only if the read-write mode fails we try to readonly.
*/
try_open_mode= (open_flags & HA_OPEN_FORCE_MODE) ? mode : O_RDWR;
if ((kfile= mysql_file_open(mi_key_file_kfile, name_buff,
(open_mode= O_RDWR) | O_SHARE | O_NOFOLLOW | O_CLOEXEC,
(open_mode= try_open_mode) | O_SHARE |
O_NOFOLLOW | O_CLOEXEC,
MYF(MY_NOSYMLINKS))) < 0)
{
if ((errno != EROFS && errno != EACCES) ||
if ((errno != EROFS && errno != EACCES) || open_mode == O_RDONLY ||
mode != O_RDONLY ||
(kfile= mysql_file_open(mi_key_file_kfile, name_buff,
(open_mode= O_RDONLY) | O_SHARE| O_NOFOLLOW | O_CLOEXEC,
(open_mode= O_RDONLY) | O_SHARE |
O_NOFOLLOW | O_CLOEXEC,
MYF(MY_NOSYMLINKS))) < 0)
goto err;
}
share->mode=open_mode;
share->index_mode= share->data_mode= open_mode;
if (open_flags & HA_OPEN_DATA_READONLY)
share->data_mode= O_RDONLY;
errpos=1;
if (mysql_file_read(kfile, (uchar*)&share->state.header, head_length,
MYF(MY_NABP)))
@ -200,7 +214,9 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
my_errno= HA_WRONG_CREATE_OPTION;
goto err;
}
share->mode|= O_NOFOLLOW; /* all symlinks are resolved by realpath() */
/* all symlinks are resolved by realpath() */
share->index_mode|= O_NOFOLLOW;
share->data_mode|= O_NOFOLLOW;
}
info_length=mi_uint2korr(share->state.header.header_length);
@ -588,7 +604,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
else
{
share= old_info->s;
if (mode == O_RDWR && share->mode == O_RDONLY)
if (mode == O_RDWR && share->index_mode == O_RDONLY)
{
my_errno=EACCES; /* Can't open in write mode */
goto err;
@ -1279,10 +1295,11 @@ active seek-positions.
int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share)
{
myf flags= MY_WME | (share->mode & O_NOFOLLOW ? MY_NOSYMLINKS: 0);
myf flags= MY_WME | (share->data_mode & O_NOFOLLOW ? MY_NOSYMLINKS: 0);
DEBUG_SYNC_C("mi_open_datafile");
info->dfile= mysql_file_open(mi_key_file_dfile, share->data_file_name,
share->mode | O_SHARE | O_CLOEXEC, MYF(flags));
share->data_mode | O_SHARE | O_CLOEXEC,
MYF(flags));
return info->dfile >= 0 ? 0 : 1;
}
@ -1291,7 +1308,8 @@ int mi_open_keyfile(MYISAM_SHARE *share)
{
if ((share->kfile= mysql_file_open(mi_key_file_kfile,
share->unique_file_name,
share->mode | O_SHARE | O_NOFOLLOW | O_CLOEXEC,
share->index_mode | O_SHARE | O_NOFOLLOW
| O_CLOEXEC,
MYF(MY_NOSYMLINKS | MY_WME))) < 0)
return 1;
return 0;

79
storage/myisam/myisamchk.c
View File

@ -144,7 +144,7 @@ enum options_mc {
OPT_READ_BUFFER_SIZE, OPT_WRITE_BUFFER_SIZE, OPT_SORT_BUFFER_SIZE,
OPT_SORT_KEY_BLOCKS, OPT_DECODE_BITS, OPT_FT_MIN_WORD_LEN,
OPT_FT_MAX_WORD_LEN, OPT_FT_STOPWORD_FILE,
OPT_MAX_RECORD_LENGTH, OPT_STATS_METHOD
OPT_MAX_RECORD_LENGTH, OPT_STATS_METHOD, OPT_ACTIVE_KEYS
};
static struct my_option my_long_options[] =
@ -207,10 +207,16 @@ static struct my_option my_long_options[] =
"Print statistics information about table that is checked.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"keys-used", 'k',
"Tell MyISAM to update only some specific keys. # is a bit mask of which keys to use. This can be used to get faster inserts.",
"Tell MyISAM to update only some specific keys. # is a bit mask of which keys to use. This can be used to get faster inserts. See also keys-active",
&check_param.keys_in_use,
&check_param.keys_in_use,
0, GET_ULL, REQUIRED_ARG, -1, 0, 0, 0, 0, 0},
{"keys-active", OPT_ACTIVE_KEYS,
"Treat all not listed keys as disabled. If used with repair, the keys "
"will be disabled permanently. The argument is a list of key numbers, "
"starting from 1, separated by ','. "
"keys-active and keys-used are two ways to do the same thing",
0, 0, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"max-record-length", OPT_MAX_RECORD_LENGTH,
"Skip rows bigger than this if myisamchk can't allocate memory to hold it",
&check_param.max_record_length,
@ -331,7 +337,7 @@ static struct my_option my_long_options[] =
REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"stats_method", OPT_STATS_METHOD,
"Specifies how index statistics collection code should treat NULLs. "
"Possible values of name are \"nulls_unequal\" (default behavior for 4.1/5.0), "
"Possible values of name are \"nulls_unequal\" (default behavior for MySQL 4.1/5.0), "
"\"nulls_equal\" (emulate 4.0 behavior), and \"nulls_ignored\".",
(char**) &myisam_stats_method_str, (char**) &myisam_stats_method_str, 0,
GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@ -341,7 +347,7 @@ static struct my_option my_long_options[] =
static void print_version(void)
{
printf("%s Ver 2.7 for %s at %s\n", my_progname, SYSTEM_TYPE,
printf("%s Ver 2.8 for %s at %s\n", my_progname, SYSTEM_TYPE,
MACHINE_TYPE);
}
@ -413,6 +419,11 @@ static void usage(void)
-k, --keys-used=# Tell MyISAM to update only some specific keys. # is a\n\
bit mask of which keys to use. This can be used to\n\
get faster inserts.\n\
--keys-active\n\
Threat all not listed keys as disabled. If used with repair, the keys\n\
will be disabled permanently. The argument is a list of key numbers,\n\
starting from 1, separated by ','\n\
keys-active and keys-used are two ways to do the same thing\n\
--create-missing-keys\n\
Create missing keys. This assumes that the data\n\
file is correct and that the the number of rows stored\n\
@ -578,6 +589,32 @@ get_one_option(const struct my_option *opt,
case 'k':
check_param.keys_in_use= (ulonglong) strtoll(argument, NULL, 10);
break;
case OPT_ACTIVE_KEYS:
if (argument == disabled_my_option)
check_param.keys_in_use= ~0LL;
else
{
const char *start;
char *end, *str_end= strend(argument);
check_param.keys_in_use= 0;
for (start= argument; *start ; start= end)
{
int error;
longlong key;
end= str_end;
key= my_strtoll10(start, &end, &error);
if (error || key > 64 || (*end && *end != ','))
{
fprintf(stderr, "Wrong argument to active-keys. Expected a list of "
"numbers like 1,2,3,4\n");
exit(1); /* Change to my_exit after merge */
}
check_param.keys_in_use|= 1LL << (key-1);
if (*end == ',')
end++;
}
}
break;
case 'm':
if (argument == disabled_my_option)
check_param.testflag&= ~T_MEDIUM;
@ -838,7 +875,10 @@ static int myisamchk(HA_CHECK *param, char * filename)
open_flags|= HA_OPEN_IGNORE_IF_LOCKED | HA_OPEN_FROM_SQL_LAYER;
else
open_flags|= HA_OPEN_ABORT_IF_LOCKED;
if (!(info=mi_open(filename, open_mode, open_flags)))
if (param->testflag & T_QUICK)
open_flags|= HA_OPEN_DATA_READONLY;
if (!(info=mi_open(filename, open_mode, open_flags | HA_OPEN_FORCE_MODE)))
{
/* Avoid twice printing of isam file name */
param->error_printed=1;
@ -926,6 +966,13 @@ static int myisamchk(HA_CHECK *param, char * filename)
DBUG_RETURN(0);
}
}
/* Don't allow disable of active auto_increment keys for repair */
if (share->base.auto_key &&
(share->state.key_map & (1LL << share->base.auto_key)) &&
param->testflag & (T_REP_ANY | T_SORT_RECORDS | T_SORT_INDEX))
check_param.keys_in_use|= (1LL << share->base.auto_key);
if ((param->testflag & (T_REP_ANY | T_STATISTICS |
T_SORT_RECORDS | T_SORT_INDEX)) &&
(((param->testflag & T_UNPACK) &&
@ -1279,18 +1326,18 @@ static void descript(HA_CHECK *param, register MI_INFO *info, char * name)
printf("Status: %s\n",buff);
if (share->base.auto_key)
{
printf("Auto increment key: %13d Last value: %13s\n",
printf("Auto increment key: %15d Last value: %13s\n",
share->base.auto_key,
llstr(share->state.auto_increment,llbuff));
}
if (share->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
printf("Checksum: %23s\n",llstr(info->state->checksum,llbuff));
printf("Checksum: %25s\n",llstr(info->state->checksum,llbuff));
if (share->options & HA_OPTION_DELAY_KEY_WRITE)
printf("Keys are only flushed at close\n");
}
printf("Data records: %13s Deleted blocks: %13s\n",
printf("Data records: %15s Deleted blocks: %18s\n",
llstr(info->state->records,llbuff),llstr(info->state->del,llbuff2));
if (param->testflag & T_SILENT)
DBUG_VOID_RETURN; /* This is enough */
@ -1298,14 +1345,14 @@ static void descript(HA_CHECK *param, register MI_INFO *info, char * name)
if (param->testflag & T_VERBOSE)
{
#ifdef USE_RELOC
printf("Init-relocation: %13s\n",llstr(share->base.reloc,llbuff));
printf("Init-relocation: %15s\n",llstr(share->base.reloc,llbuff));
#endif
printf("Datafile parts: %13s Deleted data: %13s\n",
printf("Datafile parts: %15s Deleted data: %18s\n",
llstr(share->state.split,llbuff),
llstr(info->state->empty,llbuff2));
printf("Datafile pointer (bytes):%9d Keyfile pointer (bytes):%9d\n",
printf("Datafile pointer (bytes):%11d Keyfile pointer (bytes): %13d\n",
share->rec_reflength,share->base.key_reflength);
printf("Datafile length: %13s Keyfile length: %13s\n",
printf("Datafile length: %17s Keyfile length: %18s\n",
llstr(info->state->data_file_length,llbuff),
llstr(info->state->key_file_length,llbuff2));
@ -1315,13 +1362,13 @@ static void descript(HA_CHECK *param, register MI_INFO *info, char * name)
{
if (share->base.max_data_file_length != HA_OFFSET_ERROR ||
share->base.max_key_file_length != HA_OFFSET_ERROR)
printf("Max datafile length: %13s Max keyfile length: %13s\n",
printf("Max datafile length: %15s Max keyfile length: %18s\n",
llstr(share->base.max_data_file_length-1,llbuff),
ullstr(share->base.max_key_file_length - 1, llbuff2));
}
}
printf("Recordlength: %13d\n",(int) share->base.pack_reclength);
printf("Recordlength: %15d\n",(int) share->base.pack_reclength);
if (! mi_is_all_keys_active(share->state.key_map, share->base.keys))
{
longlong2str(share->state.key_map,buff,2);
@ -1331,7 +1378,7 @@ static void descript(HA_CHECK *param, register MI_INFO *info, char * name)
puts("\ntable description:");
printf("Key Start Len Index Type");
if (param->testflag & T_VERBOSE)
printf(" Rec/key Root Blocksize");
printf(" Rec/key Root Blocksize");
(void) putchar('\n');
for (key=keyseg_nr=0, keyinfo= &share->keyinfo[0] ;
@ -1368,7 +1415,7 @@ static void descript(HA_CHECK *param, register MI_INFO *info, char * name)
else
buff[0]=0;
if (param->testflag & T_VERBOSE)
printf("%11lu %12s %10d",
printf("%11lu %14s %10d",
share->state.rec_per_key_part[keyseg_nr++],
buff,keyinfo->block_length);
(void) putchar('\n');

3
storage/myisam/myisamdef.h
View File

@ -207,7 +207,8 @@ typedef struct st_mi_isam_share
uint32 ftkeys; /* Number of full-text keys + 1 */
File kfile; /* Shared keyfile */
File data_file; /* Shared data file */
int mode; /* mode of file on open */
int index_mode; /* mode on index file on open */
int data_mode; /* mode of data file on open */
uint reopen; /* How many times reopened */
uint w_locks,r_locks,tot_locks; /* Number of read/write locks */
uint blocksize; /* blocksize of keyfile */