diff --git a/.github/workflows/almalinux-8-build.yml b/.github/workflows/almalinux-8-build.yml index 8eb4598c3..e394ba99a 100644 --- a/.github/workflows/almalinux-8-build.yml +++ b/.github/workflows/almalinux-8-build.yml @@ -47,8 +47,9 @@ jobs: dnf -y install epel-release dnf config-manager --set-enabled powertools dnf -y install gcc gcc-c++ make autoconf automake m4 \ - python39 python39-pip diffutils \ + python39 python39-pip diffutils strace \ openssl openssl-devel \ + kernel-headers glibc-devel \ attr libattr-devel acl libacl-devel \ zstd libzstd-devel \ lz4 lz4-devel \ diff --git a/.gitignore b/.gitignore index f73d65f27..970908229 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,11 @@ aclocal.m4 .deps /*.exe *.dSYM/ +simdtest +t_chmod_secure +t_secure_relpath +testsuite/__pycache__/ +testsuite/chown-fake_test.py +testsuite/devices-fake_test.py +testsuite/exclude-lsh_test.py +testsuite/xattrs-hlink_test.py diff --git a/Makefile.in b/Makefile.in index 716171db1..244f0811e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -53,7 +53,8 @@ OBJS3=progress.o pipe.o @MD5_ASM@ @ROLL_SIMD@ @ROLL_ASM@ DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o popt_OBJS= popt/popt.o popt/poptconfig.o \ popt/popthelp.o popt/poptparse.o popt/poptint.o -OBJS=$(OBJS1) $(OBJS2) $(OBJS3) $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@ +NICENESS_OBJS = niceness/ionice_string.o niceness/niceness.o niceness/niceness_syscall.o niceness/parse_arguments.o +OBJS=$(OBJS1) $(OBJS2) $(OBJS3) $(DAEMON_OBJ) $(NICENESS_OBJS) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@ TLS_OBJ = tls.o syscall.o util2.o t_stub.o lib/compat.o lib/snprintf.o lib/permstring.o lib/sysxattrs.o @BUILD_POPT@ @@ -233,7 +234,7 @@ t_clean_fname$(EXEEXT): $(T_CLEAN_FNAME_OBJ) # GNU-make-only; BSD and Solaris make expand it to nothing. t_safe_arg_main.o: main.c $(HEADERS) $(CC) -I. -I$(srcdir) $(CFLAGS) $(CPPFLAGS) -Dmain=rsync_unused_main -c $(srcdir)/main.c -o t_safe_arg_main.o -T_SAFE_ARG_OBJ = t_safe_arg.o t_safe_arg_main.o $(OBJS1_NO_MAIN) $(OBJS2) $(OBJS3) $(DAEMON_OBJ) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@ +T_SAFE_ARG_OBJ = t_safe_arg.o t_safe_arg_main.o $(OBJS1_NO_MAIN) $(OBJS2) $(OBJS3) $(DAEMON_OBJ) $(NICENESS_OBJS) $(LIBOBJ) @BUILD_ZLIB@ @BUILD_POPT@ t_safe_arg$(EXEEXT): $(T_SAFE_ARG_OBJ) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(T_SAFE_ARG_OBJ) $(LIBS) diff --git a/configure.ac b/configure.ac index 58a5cac59..4e5444143 100644 --- a/configure.ac +++ b/configure.ac @@ -13,7 +13,7 @@ AC_CHECK_HEADERS(poll.h sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h dl.h \ popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netgroup.h \ zlib.h xxhash.h openssl/md4.h openssl/md5.h zstd.h lz4.h sys/file.h \ - sys/resource.h bsd/string.h idn2.h wchar.h) + sys/resource.h bsd/string.h idn2.h wchar.h linux/ioprio.h sys/syscall.h) AC_CHECK_HEADERS([netinet/ip.h], [], [], [[#include ]]) AC_HEADER_MAJOR_FIXED @@ -1571,6 +1571,8 @@ case "$CC" in ;; esac +AC_DEFINE_UNQUOTED([COMPILE_TARGET], ["$host"], [String describing the compiled target OS and architecture]) + AC_CONFIG_FILES([Makefile lib/dummy zlib/dummy popt/dummy shconfig]) AC_OUTPUT diff --git a/main.c b/main.c index 594af0090..9aeeb709f 100644 --- a/main.c +++ b/main.c @@ -1950,8 +1950,10 @@ int main(int argc,char *argv[]) #ifdef ICONV_CONST setup_iconv(); #endif - } else if (am_daemon) + } else if (am_daemon) { + niceness_renice_and_ionice_me(); return daemon_main(); + } if (am_server && protect_args) { char buf[MAXPATHLEN]; @@ -1963,6 +1965,8 @@ int main(int argc,char *argv[]) } } + niceness_renice_and_ionice_me(); + if (argc < 1) { usage(FERROR); exit_cleanup(RERR_SYNTAX); diff --git a/niceness/ionice_string.c b/niceness/ionice_string.c new file mode 100644 index 000000000..80a5b9147 --- /dev/null +++ b/niceness/ionice_string.c @@ -0,0 +1,48 @@ +/* + * Renice or ionice the rsync process to reduce its impact on the system + * ionice_string.c - string conversion utilities for ionice + * + * Copyright (C) 2026 Michael Mess + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, visit the http://fsf.org website. + */ + +#include +#include "niceness.h" + +/* + * Get the string representation for the given ionice_value + */ +const char *intToIoniceValueString(int ionice_value) +{ + if (ionice_value<-8) return 0; + if (ionice_value>9) return 0; + return Ionice_ValueStrings[ionice_value+8]; +} + +/* + * Parse string into ionice_value. Returns 1 on success, 0 otherwise. + */ +int ioniceStringToInt(char * string, int *ionice_value) +{ + for (int i=0; Ionice_ValueStrings[i]; i++) + { + if (strncasecmp(string, Ionice_ValueStrings[i], 4) == 0) // All values are 4 chars long + { + *ionice_value=i-8; + return 1; + } + } + return 0; +} diff --git a/niceness/ionice_string.h b/niceness/ionice_string.h new file mode 100644 index 000000000..4842390f4 --- /dev/null +++ b/niceness/ionice_string.h @@ -0,0 +1,31 @@ +/* + * Renice or ionice the rsync process to reduce its impact on the system + * ionice_string.h - string conversion utilities for ionice + * + * Copyright (C) 2026 Michael Mess + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, visit the http://fsf.org website. + */ + +#include "niceness.h" + +/* + * Get the string representation for the given ionice_value + */ +const char *intToIoniceValueString(int ionice_value); + +/* + * Parse string into ionice_value. Returns 1 on success, 0 otherwise. + */ +int ioniceStringToInt(char * string, int *ionice_value); diff --git a/niceness/niceness.c b/niceness/niceness.c new file mode 100644 index 000000000..c7aaa9dd9 --- /dev/null +++ b/niceness/niceness.c @@ -0,0 +1,98 @@ +/* + * Renice or ionice the rsync process to reduce its impact on the system + * niceness.c - core niceness state holder + * + * Copyright (C) 2026 Michael Mess + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, visit the http://fsf.org website. + */ + +#include "niceness.h" + +/* + * String representation of ionice. All strings have to be 4 characters long. + */ +const char *Ionice_ValueStrings[] = { + "rt_0", + "rt_1", + "rt_2", + "rt_3", + "rt_4", + "rt_5", + "rt_6", + "rt_7", + "none", + "be_0", + "be_1", + "be_2", + "be_3", + "be_4", + "be_5", + "be_6", + "be_7", + "idle", + 0 +}; + +struct niceness niceness = { + /* If set to other than 0, be nice on the local or remote site + * Contains the nice priority to be set on the process, or 0=feature is turned off, no priority will be set. + * I have to admit, that it is not possible to set a nice value of 0 with this code, + * but in most cases 0 should be the priority of the newly started rsync process already anyway. + */ + .nice_local = 0, + .nice_remote = 0, + + /* If set to other than 0, be ionice on the local or remote site + * Currently only idle is supported for ionice when turned on. + */ + .ionice_local = 0, + .ionice_remote = 0, +}; + +/** + * Turn niceness off (no nice and no ionice) + */ +void niceness_turn_off() +{ + niceness.nice_local = 0; + niceness.nice_remote = 0; + niceness.ionice_local = 0; + niceness.ionice_remote = 0; +} + +/** + * Turn niceness on (nice and ionice set to default values for maximum niceness) + */ +void niceness_turn_on() +{ + niceness.nice_local = NICENESS_RENICE_DEFAULT_PRIO; + niceness.nice_remote = NICENESS_RENICE_DEFAULT_PRIO; + niceness.ionice_local = NICENESS_IONICE_DEFAULT_PRIO; + niceness.ionice_remote = NICENESS_IONICE_DEFAULT_PRIO; +} + +/** + * Apply nice/ionice local to this rsync process + */ +void niceness_renice_and_ionice_me() +{ + if (niceness.nice_local) { + niceness_renice_me(niceness.nice_local); + } + + if (niceness.ionice_local) { + niceness_ionice_me(niceness.ionice_local); + } +} diff --git a/niceness/niceness.h b/niceness/niceness.h new file mode 100644 index 000000000..44abfa590 --- /dev/null +++ b/niceness/niceness.h @@ -0,0 +1,129 @@ +/* + * Renice or ionice the rsync process to reduce its impact on the system + * + * Copyright (C) 2026 Michael Mess + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, visit the http://fsf.org website. + */ + +#ifndef NICENESS_H // Ensure that this header file is never included more than once, thus avoiding errors with duplicate definitions +#define NICENESS_H 1 + +/* + * Enum for ionice values. + * Constants with negative numbers need root permission, others can be set by any user. + * RT_X is realtime priority + * BE_X is best effort with the given level + * IDLE means the process gets served only when no other processes are using disk io. + * NONE means best effort with level calculated by the formula (cpu_nice + 20) / 5 + * and is the default that does not need to be set. + */ +enum Ionice_Values { + RT_0 =-8, + RT_1 =-7, + RT_2 =-6, + RT_3 =-5, + RT_4 =-4, + RT_5 =-3, + RT_6 =-2, + RT_7 =-1, + NONE = 0, + BE_0 = 1, + BE_1 = 2, + BE_2 = 3, + BE_3 = 4, + BE_4 = 5, + BE_5 = 6, + BE_6 = 7, + BE_7 = 8, + IDLE = 9 +}; + +/* + * String representation of ionice. All strings have to be 4 characters long. + */ +extern const char *Ionice_ValueStrings[]; +/* +const char *Ionice_ValueStrings[] = { + "rt_0", + "rt_1", + "rt_2", + "rt_3", + "rt_4", + "rt_5", + "rt_6", + "rt_7", + "none", + "be_0", + "be_1", + "be_2", + "be_3", + "be_4", + "be_5", + "be_6", + "be_7", + "idle", + 0 +}; +*/ + +/* + * Constants to be used when no specific argument is given + */ +#define NICENESS_RENICE_DEFAULT_PRIO 19; +#define NICENESS_IONICE_DEFAULT_PRIO IDLE; + +/* + * Struct to hold state for niceness + */ +struct niceness { + /* If set to other than 0, be nice on the local or remote site + * Contains the nice priority to be set on the process, or 0=feature is turned off, no priority will be set. + * I have to admit, that it is not possible to set a nice value of 0 with this code, + * but in most cases 0 should be the priority of the newly started rsync process already anyway. + */ + int nice_local; + int nice_remote; + + /* If set to other than 0, be ionice on the local or remote site + * Currently only idle is supported for ionice when turned on. + */ + int ionice_local; + int ionice_remote; +}; + +extern struct niceness niceness; + + +/* + * Some methods to simply turn niceness on or off or configure with an argument + */ +void niceness_turn_off(); +void niceness_turn_on(); +int niceness_parse_argument(const char * arg); +int niceness_local_wants_elevated_priorities(); + +/* + * Apply nice and ionice on this local process + */ +void niceness_renice_and_ionice_me(); +void niceness_renice_me(int prio); +void niceness_ionice_me(int ionice_value); + +/* + * Utility function to get the string representation of an ionice value + */ +const char *intToIoniceValueString(int ionice_value); + +#endif // ifndef NICENESS_H diff --git a/niceness/niceness_syscall.c b/niceness/niceness_syscall.c new file mode 100644 index 000000000..af5d14e53 --- /dev/null +++ b/niceness/niceness_syscall.c @@ -0,0 +1,85 @@ +/* + * Renice or ionice the rsync process to reduce its impact on the system + * niceness_syscall.c - renice/ionice system call + * + * Copyright (C) 2026 Michael Mess + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, visit the http://fsf.org website. + */ + +#include "rsync.h" + +void niceness_renice_me(int prio) +{ +#ifdef SUPPORT_RENICE + int which = PRIO_PROCESS; // who specifies a Process ID + int who = 0; // 0 means current process + int result = setpriority(which, who, prio); + if ( result < 0 ) { + // Failed to set priority, inform user, but can be ignored (it's just not so nice). + rprintf(FWARNING, "renice %s to new priority %d failed (%s version %s): %s\n", + client_or_server_string(), prio, RSYNC_NAME, rsync_version(), strerror(errno)); + } else { + if (DEBUG_GTE(CMD, 1)) + rprintf(FINFO, "successfully reniced %s to new priority %d\n", client_or_server_string(), prio); + } +#else + rprintf(FWARNING, "renice %s to new priority %d failed (%s version %s): renice not supported for %s\n", + client_or_server_string(), prio, RSYNC_NAME, rsync_version(), COMPILE_TARGET); +#endif +} + +void niceness_ionice_me(int ionice_value) +{ + const char *ionice_string = intToIoniceValueString(ionice_value); +#ifdef SUPPORT_IONICE + int which = IOPRIO_WHO_PROCESS; // who specifies a Process ID + int who = 0; // 0 means current process + int class; + int data; // Ignored when using the IOPRIO_CLASS_IDLE or IOPRIO_CLASS_NONE class + switch (ionice_string[0]) + { + case 'r': // Realtime + class = IOPRIO_CLASS_RT; + data = ionice_string[3]-'0'; // rt_X: X -> data + break; + case 'b': // Best effort + class = IOPRIO_CLASS_BE; + data = ionice_string[3]-'0'; // be_X: X -> data + break; + case 'i': // Idle + class = IOPRIO_CLASS_IDLE; + data = 0; + break; + case 'n': // None + default: + class = IOPRIO_CLASS_NONE; + data = 0; + break; + } + int ioprio = IOPRIO_PRIO_VALUE(class, data); + int result = syscall(SYS_ioprio_set, which, who, ioprio); + if ( result < 0 ) { + // Failed to set priority, inform user, but can be ignored (it's just not so ionice). + rprintf(FWARNING, "ionice %s to new priority %s failed (%s version %s): %s\n", + client_or_server_string(), ionice_string, RSYNC_NAME, rsync_version(), strerror(errno)); + } else { + if (DEBUG_GTE(CMD, 1)) + rprintf(FINFO, "successfully ioniced %s to new priority %s\n", client_or_server_string(), ionice_string); + } +#else + rprintf(FWARNING, "ionice %s to new priority %s failed (%s version %s): ionice not supported for %s\n", + client_or_server_string(), ionice_string, RSYNC_NAME, rsync_version(), COMPILE_TARGET); +#endif +} diff --git a/niceness/parse_arguments.c b/niceness/parse_arguments.c new file mode 100644 index 000000000..d6b4de7e0 --- /dev/null +++ b/niceness/parse_arguments.c @@ -0,0 +1,183 @@ +/* + * Renice or ionice the rsync process to reduce its impact on the system + * parse_arguments.c - parse command line argument + * + * Copyright (C) 2026 Michael Mess + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, visit the http://fsf.org website. + */ + +#include "rsync.h" +#include "ionice_string.h" + +/* + * Try to parse the location and set the variables isLocal and isRemote. + * Return the pointer to the string after the found string. + */ +char *parse_location(char *str, int *isLocal, int *isRemote) +{ + if (strncasecmp(str, "local", 5) == 0) + { + *isLocal=1; + *isRemote=0; + return str+5; + } + if (strncasecmp(str, "remote", 6) == 0) + { + *isLocal=0; + *isRemote=1; + return str+6; + } + if (strncasecmp(str, "all", 3) == 0) + { + *isLocal=1; + *isRemote=1; + return str+3; + } + *isLocal=0; + *isRemote=0; + return str; +} + +char *parse_nice_value(char *str, int *nice_value) +{ + if (!str || !str[0]) return 0; // ERROR + char *pointer = str; + long nice_long = strtol(str, &pointer, 10); + if (nice_long<-20 || nice_long>=20) return 0; + *nice_value=(int)nice_long; + return pointer; +} + +char *parse_ionice_value(char *str, int *ionice_value) +{ + if (ioniceStringToInt(str, ionice_value)) { + return str+4; + } + *ionice_value=0; + return str; +} + +char *parse_nice_and_ionice_values(char *str, int *nice_value, int *ionice_value) +{ + char *pointer = parse_ionice_value(str, ionice_value); + if (pointer != str) // ionice value has been given standalone, no nice + { + *nice_value=0; + return pointer; + } + pointer = parse_nice_value(pointer, nice_value); + if (pointer == 0) return 0; // ERROR + if (pointer[0] == '/') // nice/ionice value + { + pointer++; + char *pointer_after_slash = pointer; + pointer = parse_ionice_value(pointer, ionice_value); + if (pointer == pointer_after_slash) // required valid ionice value missing after slash + { + return 0; + } + } + return pointer; +} + +/* + * parse the configuration string and return a pointer to the still not processed remainder of the configuration string + */ +char *parse_setting(char *config_str, int *nice_local, int *ionice_local, int *nice_remote, int *ionice_remote) +{ + int isLocal=0; + int isRemote=0; + char *pointer=parse_location(config_str, &isLocal, &isRemote); + if (isLocal || isRemote) + { /* Location "local" or "remote" or "all" found */ + if (pointer[0] == ':') + { /* Location followed by specific setting */ + pointer++; /* Skip colon */ + /* Read nice/ionice value */ + int nice_value=0; + int ionice_value=0; + pointer = parse_nice_and_ionice_values(pointer, &nice_value, &ionice_value); + if (isLocal) { + *nice_local=nice_value; + *ionice_local=ionice_value; + } + if (isRemote) { + *nice_remote=nice_value; + *ionice_remote=ionice_value; + } + return pointer; + } else { + if (!pointer[0] || pointer[0] == ',') + { /* End of string or comma */ + /* Location without specific setting, use default */ + if (isLocal) { + *nice_local = NICENESS_RENICE_DEFAULT_PRIO; + *ionice_local = NICENESS_IONICE_DEFAULT_PRIO; + } + if (isRemote) { + *nice_remote = NICENESS_RENICE_DEFAULT_PRIO; + *ionice_remote = NICENESS_IONICE_DEFAULT_PRIO; + } + return pointer; + } else + { /* Unexpected characters */ + return 0; // ERROR + } + } + } else { + /* Read nice/ionice value */ + int nice_value=0; + int ionice_value=0; + pointer = parse_nice_and_ionice_values(pointer, &nice_value, &ionice_value); + *nice_local=nice_value; + *ionice_local=ionice_value; + *nice_remote=nice_value; + *ionice_remote=ionice_value; + return pointer; + } + return 0; // ERROR +} + + +int parse_argument(const char *config_str, int *nice_local, int *ionice_local, int *nice_remote, int *ionice_remote) +{ + char *str = (char *) config_str; + while (str && str[0]) { // string is valid and not empty + str=parse_setting(str, nice_local, ionice_local, nice_remote, ionice_remote); + if (!str) return 0; // ERROR returned by parser + if (!str[0]) return 1; // End of string - everything has been parsed - finished successfully! + if (str[0] != ',') return 0; // ERROR: Unexpected character, we expect a comma here as separator. + str++; + } + return 0; // string was empty or not valid (maybe even after a comma) +} + + +/** + * Parse the given argument and set nice and ionice accordingly + */ +int niceness_parse_argument(const char * arg) +{ + return parse_argument(arg, &niceness.nice_local, &niceness.ionice_local, &niceness.nice_remote, &niceness.ionice_remote); +} + + +/** + * Check nice/ionice already parsed before for locally elevated priority values + */ +int niceness_local_wants_elevated_priorities() +{ + return niceness.nice_local < 0 || niceness.ionice_local < 0; +} diff --git a/options.c b/options.c index 90e94bb7c..d41d929bf 100644 --- a/options.c +++ b/options.c @@ -36,6 +36,7 @@ extern filter_rule_list daemon_filter_list; int make_backups = 0; + /** * If 1, send the whole file as literal data rather than trying to * create an incremental diff. @@ -604,7 +605,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM, OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE, OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR, OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, OPT_OLD_ARGS, - OPT_STOP_AFTER, OPT_STOP_AT, + OPT_STOP_AFTER, OPT_STOP_AT, OPT_NICE, OPT_NO_NICE, OPT_REFUSED_BASE = 9000}; static struct poptOption long_options[] = { @@ -862,6 +863,9 @@ static struct poptOption long_options[] = { {"remote-option", 'M', POPT_ARG_STRING, 0, 'M', 0, 0 }, {"protocol", 0, POPT_ARG_INT, &protocol_version, 0, 0, 0 }, {"checksum-seed", 0, POPT_ARG_INT, &checksum_seed, 0, 0, 0 }, + {"nice", 0, POPT_ARG_STRING, 0, OPT_NICE, 0, 0 }, + {"no-nice", 0, POPT_ARG_NONE, 0, OPT_NO_NICE, 0, 0 }, + {0, 'Q', POPT_ARG_NONE, 0, 'Q', 0, 0 }, {"server", 0, POPT_ARG_NONE, 0, OPT_SERVER, 0, 0 }, {"sender", 0, POPT_ARG_NONE, 0, OPT_SENDER, 0, 0 }, /* All the following options switch us into daemon-mode option-parsing. */ @@ -1467,6 +1471,14 @@ char *alt_dest_opt(int type) } } +/** + * Returns a string representation for the current function as client or server + */ +const char * client_or_server_string() +{ + return am_server ? "server" : "client"; +} + /** * Process command line arguments. Called on both local and remote. * @@ -1714,6 +1726,43 @@ int parse_arguments(int *argc_p, const char ***argv_p) quiet++; break; + case 'Q': + /* + * Turn nice on with default prio and turn ionice on (idle). + */ + niceness_turn_on(); + break; + + case OPT_NICE: + /* + * Parse parameters for nice and set the following fields: + - nice_local + - ionice_local + - nice_remote + - ionice_remote + */ + arg = poptGetOptArg(pc); + if (!niceness_parse_argument(arg)) { + snprintf(err_buf, sizeof err_buf, + "Invalid argument passed to --nice (%s)\n", + arg); + goto cleanup; + } + if (am_daemon && niceness_local_wants_elevated_priorities()) { + snprintf(err_buf, sizeof err_buf, + "Elevated or real-time priorities rejected by rsync daemon (%s)\n", + arg); + goto cleanup; + } + break; + + case OPT_NO_NICE: + /* + * Turn off nice and ionice + */ + niceness_turn_off(); + break; + case 'x': one_file_system++; break; @@ -3199,6 +3248,14 @@ void server_options(char **args, int *argc_p) if (mkpath_dest_arg && am_sender) args[ac++] = "--mkpath"; + if (niceness.nice_remote || niceness.ionice_remote) { + const char *ionice_str = niceness.ionice_remote ? intToIoniceValueString(niceness.ionice_remote):""; + const char *slash_str = niceness.ionice_remote ? "/":""; + if (asprintf(&arg, "--nice=local:%d%s%s", niceness.nice_remote, slash_str, ionice_str) < 0) + goto oom; + args[ac++] = arg; + } + if (ac > MAX_SERVER_ARGS) { /* Not possible... */ rprintf(FERROR, "argc overflow in server_options().\n"); exit_cleanup(RERR_MALLOC); diff --git a/rsync.1.md b/rsync.1.md index 415480c8a..19d97b4d5 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -686,6 +686,15 @@ has its own detailed description later in this manpage. --checksum-seed=NUM set block/file checksum seed (advanced) --ipv4, -4 prefer IPv4 --ipv6, -6 prefer IPv6 +--nice=(LOCATION|[LOCATION:]SETTING)[,...] Set prio of the rsync process + LOCATION: local|remote|all + SETTING: NICESETTING|IONICESETTING|NICESETTING/IONICESETTING + NICESETTING: -20...19 + IONICESETTING: RT_0|RT_1|RT_2|RT_3|RT_4|RT_5|RT_6|RT_7|NONE| + BE_0|BE_1|BE_2|BE_3|BE_4|BE_5|BE_6|BE_7|IDLE +--no-nice prepending "no-" turns the option off +-Q same as --nice=all:19/idle + LOCATION and IONICESETTING are not case sensitive --version, -V print the version + other info and exit --help, -h (*) show this help (* -h is help only on its own) ``` @@ -706,6 +715,7 @@ accepted: --log-file=FILE override the "log file" setting --log-file-format=FMT override the "log format" setting --sockopts=OPTIONS specify custom TCP options +--nice=... set priority of the local daemon process --verbose, -v increase verbosity --ipv4, -4 prefer IPv4 --ipv6, -6 prefer IPv6 @@ -4131,6 +4141,76 @@ sign) if you want the local shell to expand it. user wants a more random checksum seed. Setting NUM to 0 causes rsync to use the default of **time**() for checksum seed. +0. `--nice=(LOCATION|[LOCATION:]SETTING)[,...]` + + LOCATION: `LOCAL`|`REMOTE`|`ALL` + + - `LOCAL` is the local rsync process that has been started by the user/script/etc. + + - `REMOTE` it the remote rsync process that has been started via SSH on the remote system + + - `ALL` means both, local and remote. + The constants are not case sensitive, thus e.g. `REMOTE` and `remote` are the same value. + If omitted, the default is: all + + SETTING: NICESETTING|IONICESETTING|NICESETTING/IONICESETTING + - If the whole setting is omitted, the default is `19/IDLE` + + NICESETTING: `-20`...`19` + - Higher number means lower priority, negative numbers require root permissions + - If not specified in the setting, the default is `0`. + - No system call is made to set the priority `0`. + - If the setting fails, e.g. due to the lack of permission to set negative nice + priority, a warning is issued and rsync will continue doing its work. + - Setting elevated priorities (<0) on a remote daemon process is rejected for security reasons. + + IONICESETTING: RT_0|RT_1|RT_2|RT_3|RT_4|RT_5|RT_6|RT_7|NONE|BE_0|BE_1|BE_2|BE_3|BE_4|BE_5|BE_6|BE_7|IDLE + - `RT_X` is realtime priority, need root permission + - `BE_X` is best effort with the given level + - `IDLE` means the process gets served only when no other processes are using disk io. + - `NONE` means best effort with level calculated by the formula (cpu_nice + 20) / 5 + and is the default that does not need to be set. + - The constants are not case sensitive, thus e.g. `IDLE` and `idle` are the same value. + - If not specified in the setting, the default is `NONE`. + - No system call is made to set the priority `NONE`. + - If the setting fails, e.g. due to the lack of permission to set realtime + priority, a warning is issued and rsync will continue doing its work. + - Setting real-time priorities (RT_X) on a remote daemon process is rejected for security reasons. + + Examples: + + Specifying a setting for both, local and remote: + --nice=19/idle = --nice=all:19/idle + --nice=19 = --nice=all:19 (no ionice -> NONE => only nice will be set) + --nice=idle = --nice=all:idle (no nice -> 0 => only ionice will be set) + --nice=0 = --nice=all:0 (nice value 0 and no ionice -> 0/NONE => none of them is set) + + Specifying a setting for the given location: + --nice=local:10 + --nice=remote:idle + --nice=local:19,remote:5 + --nice=local:19/idle,remote:5 + --nice=local:19/be_0,remote:idle + + No specific setting means default 19/idle: + --nice=local = --nice=local:19/idle + --nice=remote = --nice=remote:19/idle + --nice=all = --nice=19/idle + -Q = --nice=19/idle + --nice=local,remote + + Any specific setting set before may be overwritten by the following setting, e. g.: + --nice=local:5 --nice=all + --nice=local:6/idle,all + --nice=local:5 --nice=remote:6 -Q + would result in setting the default 19/idle for both, local and remote. + + nice and ionice are always set as a pair, even when only one of them is specified. + --nice=remote:5 --nice=remote:idle + therefore would result in an unset nice value (`5` will be overwritten with default `0`) + and effectively only ionice value `idle` beeing applied. + + ## DAEMON OPTIONS The options allowed when starting an rsync daemon are as follows: diff --git a/rsync.h b/rsync.h index d90c061a5..e5402c27a 100644 --- a/rsync.h +++ b/rsync.h @@ -18,6 +18,9 @@ * with this program; if not, visit the http://fsf.org website. */ +#ifndef RSYNC_H // Ensure that this header file is never included more than once, thus avoiding errors with duplicate definitions +#define RSYNC_H 1 + #define False 0 #define True 1 #define Unset (-1) /* Our BOOL values are always an int. */ @@ -1546,3 +1549,19 @@ const char *get_panic_action(void); #elif defined HAVE_MALLINFO #define MEM_ALLOC_INFO mallinfo #endif + +/* Some header files needed to be nice ;) */ +#include "niceness/niceness.h" + +#ifdef HAVE_SYS_RESOURCE_H +#define SUPPORT_RENICE +#include +#endif + +#if defined HAVE_LINUX_IOPRIO_H && defined HAVE_SYS_SYSCALL_H +#define SUPPORT_IONICE +#include +#include +#endif + +#endif // ifndef RSYNC_H diff --git a/testsuite/niceness-option-processing_test.py b/testsuite/niceness-option-processing_test.py new file mode 100644 index 000000000..c3531c9c0 --- /dev/null +++ b/testsuite/niceness-option-processing_test.py @@ -0,0 +1,476 @@ +#!/usr/bin/env python3 +# Python handling of niceness options like --nice or --ionice. +# +# Foundational smoke test: --version / --info=help / --debug=help all +# work with --nice or --ionice without crash, +# plus test passing the correct options to the remote command + +import os + +from rsyncfns import ( + FROMDIR, RSYNC, SRCDIR, TODIR, + checkit, run_rsync, test_fail, test_skipped +) + +# Helper function +def fail(msg: str) -> 'None': + print('### TEST FAILURE ### ' + msg + '\n') + test_fail(msg) + +# Basic help dumps must not crash when using any niceness option. +if run_rsync('-Q', '--version', check=False).returncode != 0: + fail('-Q --version output failed') +if run_rsync('-Q', '--info=help', check=False).returncode != 0: + fail('-Q --info=help output failed') +if run_rsync('-Q', '--debug=help', check=False).returncode != 0: + fail('-Q --debug=help output failed') + +# We just want to test which parameters are handed over to the remote, +# but do not really want to run rsync on the remote site to copy files. +# For this we add "--rsh=echo" to replace ssh for the remote site with echo. +# Thus no remote connection is actually created, instead the echo will just +# exit quickly without any further rsync communication, letting the local +# rsync process fail fast, not wasting much time for the test execution. +# It is expected that the local rsync process will exit here with non zero. +# We then just check the log output here for any options that may or may not +# have been given for the remote rsync process by our local rsync process. +def test_remote_args(testname: str, args: list, expected_nice_value: str, expected_ionice_value: str, expected_remote_args: list) -> None: + print('### TEST BEGIN ### ' + testname + '\n') + probe = run_rsync(*['-nvv', '--rsh=echo', *args], check=False, capture_output=True) + if probe.returncode == 0: + fail('The command should not succeed here.') + actual_remote_args = 'not found' + nice_value="" + ionice_value="" + for remote_line in probe.stdout.splitlines(): + if 'opening connection using' in remote_line: + actual_remote_args = remote_line.strip() + elif 'client to new priority' in remote_line: + print("stdout: Found client info: "+remote_line); + if 'renice' in remote_line: + nice_value = remote_line.split("to new priority", 1)[1].strip() + print("Local nice value: "+nice_value); + else: + ionice_value = remote_line.split("to new priority", 1)[1].strip() + print("Local ionice value: "+ionice_value); + else: + print('ignoring line: ' + remote_line) + for remote_line in probe.stderr.splitlines(): + if 'client to new priority' in remote_line: + print("stderr: Found client info: "+remote_line); + if 'renice' in remote_line: + nice_value = remote_line.split("to new priority", 1)[1].split("failed", 1)[0].strip() + print("Local nice value: "+nice_value); + else: + ionice_value = remote_line.split("to new priority", 1)[1].split("failed", 1)[0].strip() + print("Local ionice value: "+ionice_value); + if 'rsync: Invalid argument passed to --nice' in remote_line: + print("stderr: rsync complains about invalid argument: "+remote_line); + nice_value = "invalid" + ionice_value = "invalid" + actual_remote_args = "none" + if 'not found' == actual_remote_args: + fail('No remote shell command has been started or it was not logged.') + if not nice_value == expected_nice_value: + fail('Local nice value does not match: expected: '+expected_nice_value+", but was: "+nice_value) + if not ionice_value == expected_ionice_value: + fail('Local ionice value does not match: expected: '+expected_ionice_value+", but was: "+ionice_value) + for string in expected_remote_args: + if string[0] == '!': + substring = string[1:] + if (substring+' ') in actual_remote_args or ('\"'+substring+'\"') in actual_remote_args: + fail('Remote command line contains unexpected ' + substring + ': ' + actual_remote_args) + else: + if not ((string+' ') in actual_remote_args or ('\"'+string+'\"') in actual_remote_args): + fail('Remote command line did not contain expected ' + string + ': ' + actual_remote_args) + print('### TEST END ### ' + testname + '\n') + +test_remote_args('Check with no flags', [f'localhost:{FROMDIR}', TODIR], "", "", ["--server", "--sender", "!--blahblah", + "!--nice", + "!--nice-local", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with -Q', ['-Q', f'localhost:{FROMDIR}', TODIR], "19", "idle", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:19/idle", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +# When the -s option is used, the options are transferred via connection, not command line +# The test cases here can just monitor the given command line, but not the transfered options +test_remote_args('Check with -s -Q', ['-s', '-Q', f'localhost:{FROMDIR}', TODIR], "19", "idle", ["--server", "--sender", "!--blahblah", + "!--nice", + "!--nice=local:19/idle", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=all', ['--nice=all', f'localhost:{FROMDIR}', TODIR], "19", "idle", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:19/idle", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=local', ['--nice=local', f'localhost:{FROMDIR}', TODIR], "19", "idle", ["--server", "--sender", "!--blahblah", + "!--nice", + "!--nice=local:19/idle", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=remote', ['--nice=remote', f'localhost:{FROMDIR}', TODIR], "", "", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:19/idle", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=idle', ['--nice=idle', f'localhost:{FROMDIR}', TODIR], "", "idle", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:0/idle", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=be_2', ['--nice=be_2', f'localhost:{FROMDIR}', TODIR], "", "be_2", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:0/be_2", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=be_5', ['--nice=be_5', f'localhost:{FROMDIR}', TODIR], "", "be_5", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:0/be_5", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=rt_3', ['--nice=rt_3', f'localhost:{FROMDIR}', TODIR], "", "rt_3", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:0/rt_3", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=rt_7', ['--nice=rt_7', f'localhost:{FROMDIR}', TODIR], "", "rt_7", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:0/rt_7", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=RT_0', ['--nice=RT_0', f'localhost:{FROMDIR}', TODIR], "", "rt_0", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:0/rt_0", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=none', ['--nice=none', f'localhost:{FROMDIR}', TODIR], "", "", ["--server", "--sender", "!--blahblah", + "!--nice", + "!--nice=local:0/none", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=NONE', ['--nice=NONE', f'localhost:{FROMDIR}', TODIR], "", "", ["--server", "--sender", "!--blahblah", + "!--nice", + "!--nice=local:0/none", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=local:idle', ['--nice=local:idle', f'localhost:{FROMDIR}', TODIR], "", "idle", ["--server", "--sender", "!--blahblah", + "!--nice", + "!--nice=local:19/idle", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=remote:idle', ['--nice=remote:idle', f'localhost:{FROMDIR}', TODIR], "", "", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:0/idle", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=local:4', ['--nice=local:4', f'localhost:{FROMDIR}', TODIR], "4", "", ["--server", "--sender", "!--blahblah", + "!--nice", + "!--nice=local:4", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=local:19', ['--nice=local:19', f'localhost:{FROMDIR}', TODIR], "19", "", ["--server", "--sender", "!--blahblah", + "!--nice", + "!--nice=local:4", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=local:20', ['--nice=local:20', f'localhost:{FROMDIR}', TODIR], "invalid", "invalid", ["!--server", "!--sender", "!--blahblah", + "!--nice", + "!--nice=local:4", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=local:21', ['--nice=local:21', f'localhost:{FROMDIR}', TODIR], "invalid", "invalid", ["!--server", "!--sender", "!--blahblah", + "!--nice", + "!--nice=local:4", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=local:-21', ['--nice=local:-21', f'localhost:{FROMDIR}', TODIR], "invalid", "invalid", ["!--server", "!--sender", "!--blahblah", + "!--nice", + "!--nice=local:4", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=local:-20', ['--nice=local:-20', f'localhost:{FROMDIR}', TODIR], "-20", "", ["--server", "--sender", "!--blahblah", + "!--nice", + "!--nice=local:4", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=remote:6', ['--nice=remote:6', f'localhost:{FROMDIR}', TODIR], "", "", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:6", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + + +test_remote_args('Check with --nice=6', ['--nice=6', f'localhost:{FROMDIR}', TODIR], "6", "", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:6", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + + +test_remote_args('Check with --nice=-2/IDLE', ['--nice=-2/IDLE', f'localhost:{FROMDIR}', TODIR], "-2", "idle", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:-2/idle", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=ALL:-2/IDLE', ['--nice=ALL:-2/IDLE', f'localhost:{FROMDIR}', TODIR], "-2", "idle", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:-2/idle", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=local:10/BE_0,remote:19/IDLE', ['--nice=local:10/BE_0,remote:19/IDLE', f'localhost:{FROMDIR}', TODIR], "10", "be_0", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:19/idle", + "!--nice-local-value=4", + "!--nice-remote", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=2/IDLE', ['--nice=2/IDLE', f'localhost:{FROMDIR}', TODIR], "2", "idle", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:2/idle", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=2/', ['--nice=2/', f'localhost:{FROMDIR}', TODIR], "invalid", "invalid", ["!--server", "!--sender", "!--blahblah", + "!--nice", + "!--nice=local:2/", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) + +test_remote_args('Check with --nice=2', ['--nice=2', f'localhost:{FROMDIR}', TODIR], "2", "", ["--server", "--sender", "!--blahblah", + "!--nice", + "--nice=local:2", + "!--nice-local-value=4", + "!--nice-local-value=6", + "!--nice-remote", + "!--nice-remote-value=4", + "!--nice-remote-value=6", + "!--ionice", + "!--ionice-local", + "!--ionice-remote", + "!-Q" + ]) diff --git a/testsuite/niceness-ssh-strace_test.py b/testsuite/niceness-ssh-strace_test.py new file mode 100644 index 000000000..525042c91 --- /dev/null +++ b/testsuite/niceness-ssh-strace_test.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python3 +# Python testing of system calls of --nice options with ssh and strace +# +# Foundational smoke test: --version / --info=help / --debug=help all +# work with --nice or --ionice without crash, +# plus test establishing a remote transfer, stracing local and remote process to check applied priorities + +import sys +import os +import re +import shlex +import shutil +import subprocess + +from rsyncfns import ( + FROMDIR, RSYNC, SRCDIR, TODIR, + checkit, hands_setup, run_rsync, test_fail, test_skipped, rsync_path_arg, rsh_cmd +) + +# Helper function +def fail(msg: str) -> 'None': + print('### TEST FAILURE ### ' + msg + '\n') + test_fail(msg) + +def skip(msg: str) -> 'None': + print('### TEST SKIPPED ### ' + msg + '\n') + test_skipped(msg) + raise SystemExit(0) + +if sys.platform.startswith(('solaris', 'sunos5')): + skip("niceness-ssh-strace test not supported on " + sys.platform) + +print(f"Now running the tests on platform: {sys.platform}") + +# Basic help dumps must not crash when using any niceness option. +if run_rsync('-Q', '--version', check=False).returncode != 0: + fail('-Q --version output failed') +if run_rsync('-Q', '--info=help', check=False).returncode != 0: + fail('-Q --info=help output failed') +if run_rsync('-Q', '--debug=help', check=False).returncode != 0: + fail('-Q --debug=help output failed') + +SSH = rsh_cmd() + +# Allow opting into real ssh via the shconfig variable, like the shell test. +if os.environ.get('rsync_enable_ssh_tests') == 'yes': + real_ssh = shutil.which('ssh') + if real_ssh: + SSH = real_ssh + +# SSH is quoted for rsync's own tokenizer (a build path with '~' or a space +# comes out in quotes), so split it the same way before exec'ing it directly. +probe = subprocess.run( + [*shlex.split(SSH), '-oBatchMode yes', 'localhost', 'echo', 'yes'], + capture_output=True, text=True, +) +if probe.stdout.strip() != 'yes': + skip("Skipping SSH tests because ssh connection to localhost not authorised") + +STRACE=shutil.which('strace') + +if STRACE is None: + skip("Skipping SSH tests because strace is not available") + + +print(f"Using remote shell: {SSH}") + + + +def split_strace_rsync_cmd(cmd: str) -> list: + """Split an rsync command string into argv, tolerating spaces in the path. + + RSYNC may be a wrapper command ('valgrind --tool=memcheck /build/rsync'), + which has to be split, or a plain path to the binary, which must not be if + it contains a space -- shlex.split() would turn '/ws test/rsync' into two + nonexistent programs. A path that exists is one word by definition, so + check that first and only fall back to splitting for a real command line. + + Call this at use time, never once at import: tests such as chown-fake + append ' --fake-super' to rsyncfns.RSYNC part-way through, and a cached + split would keep handing back the pre-mutation command. + + Here we support only calling rsync with strace, otherwise skip tests. + """ + if os.path.isfile(cmd): + return ['strace', '-e', 'trace=setpriority,getpriority,ioprio_set,ioprio_get', cmd] + # The path may be followed by options -- chown-fake and friends append + # ' --fake-super' to RSYNC -- so the whole string is no longer a filename. + # Take the longest leading run that names an existing file as the program + # and split only what follows. + for m in reversed(list(re.finditer(r'\s+', cmd))): + head = cmd[:m.start()] + if os.path.isfile(head): + return ['strace', '-e', 'trace=setpriority,getpriority,ioprio_set,ioprio_get', head] + shlex.split(cmd[m.start():]) + skip("Skipping SSH tests because strace is not available") + return shlex.split(cmd) + + + +def strace_rsync_argv(*args: str) -> list: + """Return the argv for invoking strace rsync with the given extra arguments. + + RSYNC may be a multi-word command (e.g. 'valgrind ... /build/rsync'); we + shlex-split it so subprocess sees a proper argv list. Each *args entry + is appended verbatim, so callers should pass tokens already split (no + embedded option/value joined by spaces). + """ + return split_strace_rsync_cmd(RSYNC) + list(args) + + + +def run_strace_rsync(*args: str, check: bool = True, + capture_output: bool = False) -> subprocess.CompletedProcess: + """Run strace rsync with the given arguments. + + By default, stdout/stderr inherit (so the runner captures them in the + per-test log). Set capture_output=True if the test needs to inspect the + output. If check is True (the default), a non-zero exit calls + test_fail() with the rsync command line. + """ + argv = strace_rsync_argv(*args) + if capture_output: + proc = subprocess.run(argv, capture_output=True, text=True) + else: + proc = subprocess.run(argv) + if check and proc.returncode != 0: + test_fail(f"rsync exited {proc.returncode}: {' '.join(argv)}") + return proc + +def expected_strace_result_as_root(string: str) -> str: + if string == "": + return string + return string.split("=", 1)[0]+'=0' + +def run_rsync_local_test(testname: str, args: list, expected: dict[str, str]) -> None: + + print('### TEST ' + testname + ' BEGIN ###\n') + hands_setup() + FROMDIR.mkdir(parents=True, exist_ok=True) + (FROMDIR / '1.txt').write_text("Testfile1'\n") + (FROMDIR / 'wasMitX.txt').write_bytes(b"x" * 10_000) + (FROMDIR / '2.txt').write_text("Testfile2'\n") + + probe = run_strace_rsync(*['-nvvr', '-e', SSH, *args, f'--rsync-path={rsync_path_arg()}', f'{FROMDIR}/', f'localhost:{TODIR}/'], check=False, capture_output=True) + print('stdout') + print(probe.stdout) + print('stderr') + print(probe.stderr) + + result = { + "nice_local":"", + "ionice_local":"", + "nice_remote":"", + "ionice_remote":"", + "nice_strace":"", + "ionice_strace":"" + } + + expected_result = {key: expected[key] for key in ["nice_local", "ionice_local", "nice_remote", "ionice_remote"] if key in expected} + expected_result["nice_strace"] = expected["nice_local_strace"] + expected_result["ionice_strace"] = expected["ionice_local_strace"] + + expected_result_as_root = {key: expected[key].replace(" failed", "") for key in ["nice_local", "ionice_local", "nice_remote", "ionice_remote"] if key in expected} + expected_result_as_root["nice_strace"] = expected_strace_result_as_root(expected["nice_local_strace"]) + expected_result_as_root["ionice_strace"] = expected_strace_result_as_root(expected["ionice_local_strace"]) + + for remote_line in probe.stdout.splitlines(): + print("stdout line: "+remote_line); + if 'nice not supported for' in remote_line: + skip("Skipping tests because niceness is not supported on this platform") + if 'opening connection using' in remote_line: + actual_remote_args = remote_line.strip() + elif 'client to new priority' in remote_line: + print("stdout: Found client info: "+remote_line); + if 'renice' in remote_line: + result["nice_local"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Local nice value: "+result["nice_local"]); + else: + result["ionice_local"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Local ionice value: "+result["ionice_local"]); + else: + print('ignoring stdout line: ' + remote_line) + for remote_line in probe.stderr.splitlines(): + print("stderr line: "+remote_line); + if 'nice not supported for' in remote_line: + skip("Skipping tests because niceness is not supported on this platform") + if 'Host key verification failed' in remote_line: + skip("Skipping tests because ssh could not establish connection: Host key verification failed") + if 'error creating process trace' in remote_line: + skip("Skipping tests because strace could not create a process trace") + if 'setpriority(PRIO_PROCESS, 0,' in remote_line: + result["nice_strace"] = remote_line.split("(PRIO_PROCESS, 0,", 1)[1].replace(" ", "").replace(")=", "=") + elif 'ioprio_set(IOPRIO_WHO_PROCESS, 0, IOPRIO_PRIO_VALUE' in remote_line: + result["ionice_strace"] = remote_line.split("IOPRIO_PRIO_VALUE(", 1)[1].replace(" ", "").replace("))=", "=") + elif 'client to new priority' in remote_line: + print("stdout: Found client info: "+remote_line); + if 'renice' in remote_line: + result["nice_local"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Local nice value: "+result["nice_local"]); + else: + result["ionice_local"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Local ionice value: "+result["ionice_local"]); + elif 'server to new priority' in remote_line: + print("stderr: Found server info: "+remote_line); + if 'renice' in remote_line: + result["nice_remote"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Remote nice value: "+result["nice_remote"]); + else: + result["ionice_remote"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Remote ionice value: "+result["ionice_remote"]); + elif 'rsync: Invalid argument passed to --nice' in remote_line: + print("stderr: rsync complains about invalid argument: "+remote_line); + result["rsync_fail"] = true + else: + print('ignoring stderr line: ' + remote_line) + test_result = result == expected_result or result == expected_result_as_root + if test_result: + print('### TEST ' + testname + ': SUCCESS ###\nResult : ' + str(result)) + else: + print('### TEST ' + testname + ': FAILED ###\n') + sys.stdout.flush() + fail('### TEST ' + testname +": \nExpected as user: "+ str(expected_result) + " \nbut was : " + str(result) +": \nExpected as root: "+ str(expected_result_as_root)) + print('### TEST ' + testname + ' END ###\n') + sys.stderr.flush() + sys.stdout.flush() + +def run_rsync_remote_test(testname: str, args: list, expected: dict[str, str]) -> None: + + print('### TEST ' + testname + ' BEGIN ###\n') + hands_setup() + FROMDIR.mkdir(parents=True, exist_ok=True) + (FROMDIR / '1.txt').write_text("Testfile1'\n") + (FROMDIR / 'wasMitX.txt').write_bytes(b"x" * 10_000) + (FROMDIR / '2.txt').write_text("Testfile2'\n") + + probe = run_rsync(*['-nvvr', '-e', SSH, *args, f'--rsync-path=strace -e trace=setpriority,getpriority,ioprio_set,ioprio_get {rsync_path_arg()}', f'{FROMDIR}/', f'localhost:{TODIR}/'], check=False, capture_output=True) + print('stdout') + print(probe.stdout) + print('stderr') + print(probe.stderr) + + result = { + "nice_local":"", + "ionice_local":"", + "nice_remote":"", + "ionice_remote":"", + "nice_strace":"", + "ionice_strace":"" + } + + expected_result = {key: expected[key] for key in ["nice_local", "ionice_local", "nice_remote", "ionice_remote"] if key in expected} + expected_result["nice_strace"] = expected["nice_remote_strace"] + expected_result["ionice_strace"] = expected["ionice_remote_strace"] + + expected_result_as_root = {key: expected[key].replace(" failed", "") for key in ["nice_local", "ionice_local", "nice_remote", "ionice_remote"] if key in expected} + expected_result_as_root["nice_strace"] = expected_strace_result_as_root(expected["nice_remote_strace"]) + expected_result_as_root["ionice_strace"] = expected_strace_result_as_root(expected["ionice_remote_strace"]) + + for remote_line in probe.stdout.splitlines(): + print("stdout line: "+remote_line); + if 'nice not supported for' in remote_line: + skip("Skipping tests because niceness is not supported on this platform") + if 'opening connection using' in remote_line: + actual_remote_args = remote_line.strip() + elif 'client to new priority' in remote_line: + print("stdout: Found client info: "+remote_line); + if 'renice' in remote_line: + result["nice_local"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Local nice value: "+result["nice_local"]); + else: + result["ionice_local"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Local ionice value: "+result["ionice_local"]); + else: + print('ignoring stdout line: ' + remote_line) + for remote_line in probe.stderr.splitlines(): + print("stderr line: "+remote_line); + if 'nice not supported for' in remote_line: + skip("Skipping tests because niceness is not supported on this platform") + if 'Host key verification failed' in remote_line: + skip("Skipping tests because ssh could not establish connection: Host key verification failed") + if 'error creating process trace' in remote_line: + skip("Skipping tests because strace could not create a process trace") + if 'setpriority(PRIO_PROCESS, 0,' in remote_line: + result["nice_strace"] = remote_line.split("(PRIO_PROCESS, 0,", 1)[1].replace(" ", "").replace(")=", "=") + elif 'ioprio_set(IOPRIO_WHO_PROCESS, 0, IOPRIO_PRIO_VALUE' in remote_line: + result["ionice_strace"] = remote_line.split("IOPRIO_PRIO_VALUE(", 1)[1].replace(" ", "").replace("))=", "=") + elif 'client to new priority' in remote_line: + print("stdout: Found client info: "+remote_line); + if 'renice' in remote_line: + result["nice_local"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Local nice value: "+result["nice_local"]); + else: + result["ionice_local"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Local ionice value: "+result["ionice_local"]); + elif 'server to new priority' in remote_line: + print("stderr: Found server info: "+remote_line); + if 'renice' in remote_line: + result["nice_remote"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Remote nice value: "+result["nice_remote"]); + else: + result["ionice_remote"] = remote_line.split("to new priority", 1)[1].split("(", 1)[0].strip() + print("Remote ionice value: "+result["ionice_remote"]); + elif 'rsync: Invalid argument passed to --nice' in remote_line: + print("stderr: rsync complains about invalid argument: "+remote_line); + result["rsync_fail"] = true + else: + print('ignoring stderr line: ' + remote_line) + test_result = result == expected_result or result == expected_result_as_root + if test_result: + print('### TEST ' + testname + ': SUCCESS ###\nResult : ' + str(result)) + else: + print('### TEST ' + testname + ': FAILED ###\n') + sys.stdout.flush() + fail('### TEST ' + testname +": \nExpected as user: "+ str(expected_result) + " \nbut was : " + str(result) +": \nExpected as root: "+ str(expected_result_as_root)) + print('### TEST ' + testname + ' END ###\n') + sys.stderr.flush() + sys.stdout.flush() + + +def run_rsync_nice_test(testname: str, args: list, expected_result: dict[str, str]) -> None: + run_rsync_local_test(testname+' (local strace)', args, expected_result) + run_rsync_remote_test(testname+' (remote strace)', args, expected_result) + +run_rsync_nice_test("Test with real rsync, no options", [], { + "nice_local":"", + "ionice_local":"", + "nice_remote":"", + "ionice_remote":"", + "nice_local_strace": "", + "ionice_local_strace": "", + "nice_remote_strace": "", + "ionice_remote_strace": ""}); + +run_rsync_nice_test("Test with real rsync -Q", [ '-Q' ], { + "nice_local":"19", + "ionice_local":"idle", + "nice_remote":"19", + "ionice_remote":"idle", + "nice_local_strace": "19=0", + "ionice_local_strace": "IOPRIO_CLASS_IDLE,0=0", + "nice_remote_strace": "19=0", + "ionice_remote_strace": "IOPRIO_CLASS_IDLE,0=0"}); + +run_rsync_nice_test("Test with real rsync -s -Q", [ '-s', '-Q' ], { + "nice_local":"19", + "ionice_local":"idle", + "nice_remote":"19", + "ionice_remote":"idle", + "nice_local_strace": "19=0", + "ionice_local_strace": "IOPRIO_CLASS_IDLE,0=0", + "nice_remote_strace": "19=0", + "ionice_remote_strace": "IOPRIO_CLASS_IDLE,0=0"}); + +run_rsync_nice_test("Test with real rsync nice local", [ '--nice=local' ], { + "nice_local":"19", + "ionice_local":"idle", + "nice_remote":"", + "ionice_remote":"", + "nice_local_strace": "19=0", + "ionice_local_strace": "IOPRIO_CLASS_IDLE,0=0", + "nice_remote_strace": "", + "ionice_remote_strace": ""}); + +run_rsync_nice_test("Test with real rsync nice remote", [ '--nice=remote' ], { + "nice_local":"", + "ionice_local":"", + "nice_remote":"19", + "ionice_remote":"idle", + "nice_local_strace": "", + "ionice_local_strace": "", + "nice_remote_strace": "19=0", + "ionice_remote_strace": "IOPRIO_CLASS_IDLE,0=0"}); + +run_rsync_nice_test("Test with real rsync nice local successfully", [ '--nice=local:5/be_6' ], { + "nice_local":"5", + "ionice_local":"be_6", + "nice_remote":"", + "ionice_remote":"", + "nice_local_strace": "5=0", + "ionice_local_strace": "IOPRIO_CLASS_BE,6=0", + "nice_remote_strace": "", + "ionice_remote_strace": ""}); + +run_rsync_nice_test("Test with real rsync nice remote successfully", [ '--nice=remote:5/be_6' ], { + "nice_local":"", + "ionice_local":"", + "nice_remote":"5", + "ionice_remote":"be_6", + "nice_local_strace": "", + "ionice_local_strace": "", + "nice_remote_strace": "5=0", + "ionice_remote_strace": "IOPRIO_CLASS_BE,6=0"}); + +run_rsync_nice_test("Test with real rsync nice local+remote successfully", [ '--nice=local:1/be_2,remote:5/be_6' ], { + "nice_local":"1", + "ionice_local":"be_2", + "nice_remote":"5", + "ionice_remote":"be_6", + "nice_local_strace": "1=0", + "ionice_local_strace": "IOPRIO_CLASS_BE,2=0", + "nice_remote_strace": "5=0", + "ionice_remote_strace": "IOPRIO_CLASS_BE,6=0"}); + +run_rsync_nice_test("Test with real rsync nice local+remote successfully with -s ", [ '--nice=local:1/be_2,remote:5/be_6', '-s' ], { + "nice_local":"1", + "ionice_local":"be_2", + "nice_remote":"5", + "ionice_remote":"be_6", + "nice_local_strace": "1=0", + "ionice_local_strace": "IOPRIO_CLASS_BE,2=0", + "nice_remote_strace": "5=0", + "ionice_remote_strace": "IOPRIO_CLASS_BE,6=0"}); + +run_rsync_nice_test("Test with real rsync nice local (nice rejected)", [ '--nice=local:-4/idle' ], { + "nice_local":"-4 failed", + "ionice_local":"idle", + "nice_remote":"", + "ionice_remote":"", + "nice_local_strace": "-4=-1EACCES(Permissiondenied)", + "ionice_local_strace": "IOPRIO_CLASS_IDLE,0=0", + "nice_remote_strace": "", + "ionice_remote_strace": ""}); + +run_rsync_nice_test("Test with real rsync nice remote (nice rejected)", [ '--nice=remote:-4/be_0' ], { + "nice_local":"", + "ionice_local":"", + "nice_remote":"-4 failed", + "ionice_remote":"be_0", + "nice_local_strace": "", + "ionice_local_strace": "", + "nice_remote_strace": "-4=-1EACCES(Permissiondenied)", + "ionice_remote_strace": "IOPRIO_CLASS_BE,0=0"}); + +run_rsync_nice_test("Test with real rsync nice local (ionice rejected)", [ '--nice=local:5/rt_6' ], { + "nice_local":"5", + "ionice_local":"rt_6 failed", + "nice_remote":"", + "ionice_remote":"", + "nice_local_strace": "5=0", + "ionice_local_strace": "IOPRIO_CLASS_RT,6=-1EPERM(Operationnotpermitted)", + "nice_remote_strace": "", + "ionice_remote_strace": ""}); + +run_rsync_nice_test("Test with real rsync nice remote (ionice rejected)", [ '--nice=remote:5/rt_6' ], { + "nice_local":"", + "ionice_local":"", + "nice_remote":"5", + "ionice_remote":"rt_6 failed", + "nice_local_strace": "", + "ionice_local_strace": "", + "nice_remote_strace": "5=0", + "ionice_remote_strace": "IOPRIO_CLASS_RT,6=-1EPERM(Operationnotpermitted)"}); + +run_rsync_nice_test("Test with real rsync nice local+remote (all rejected)", [ '--nice=local:-2/rt_3,remote:-4/rt_5' ], { + "nice_local":"-2 failed", + "ionice_local":"rt_3 failed", + "nice_remote":"-4 failed", + "ionice_remote":"rt_5 failed", + "nice_local_strace": "-2=-1EACCES(Permissiondenied)", + "ionice_local_strace": "IOPRIO_CLASS_RT,3=-1EPERM(Operationnotpermitted)", + "nice_remote_strace": "-4=-1EACCES(Permissiondenied)", + "ionice_remote_strace": "IOPRIO_CLASS_RT,5=-1EPERM(Operationnotpermitted)"}); + +run_rsync_nice_test("Test with real rsync nice local+remote (all rejected, -s)", [ '--nice=local:-2/rt_3,remote:-4/rt_5', '-s' ], { + "nice_local":"-2 failed", + "ionice_local":"rt_3 failed", + "nice_remote":"-4 failed", + "ionice_remote":"rt_5 failed", + "nice_local_strace": "-2=-1EACCES(Permissiondenied)", + "ionice_local_strace": "IOPRIO_CLASS_RT,3=-1EPERM(Operationnotpermitted)", + "nice_remote_strace": "-4=-1EACCES(Permissiondenied)", + "ionice_remote_strace": "IOPRIO_CLASS_RT,5=-1EPERM(Operationnotpermitted)"}); diff --git a/testsuite/skiplist/cygwin.txt b/testsuite/skiplist/cygwin.txt index fb822fe30..08cdd142b 100644 --- a/testsuite/skiplist/cygwin.txt +++ b/testsuite/skiplist/cygwin.txt @@ -44,6 +44,7 @@ insecure-links-admin-optout link-dest-symlink-enotsup # the ENOTSUP hard-link hook is an LD_PRELOAD, Linux-only log-file-symlink msg-io-timeout-overflow +niceness-ssh-strace # Skipping tests because strace cannot create a process trace nondaemon-symlink-race nonroot-restrictive-perms open-noatime diff --git a/testsuite/skiplist/macos.txt b/testsuite/skiplist/macos.txt index a7516cfda..b5446e093 100644 --- a/testsuite/skiplist/macos.txt +++ b/testsuite/skiplist/macos.txt @@ -18,6 +18,7 @@ daemon-auth-group dir-sgid fake-super-acl-xattr link-dest-symlink-enotsup # the ENOTSUP hard-link hook is an LD_PRELOAD, Linux-only +niceness-ssh-strace # Skipping SSH tests because strace is not available open-noatime partial-protected-regular-retry-linux preallocate