diff options
| -rw-r--r-- | doc/make.texi | 36 | ||||
| -rw-r--r-- | job.c | 172 | ||||
| -rw-r--r-- | job.h | 14 | ||||
| -rw-r--r-- | main.c | 34 | ||||
| -rw-r--r-- | makeint.h | 6 | ||||
| -rw-r--r-- | misc.c | 12 | ||||
| -rw-r--r-- | tests/scripts/features/output-sync (renamed from tests/scripts/features/parallel-sync) | 42 |
7 files changed, 154 insertions, 162 deletions
diff --git a/doc/make.texi b/doc/make.texi index 1a760e1..e004984 100644 --- a/doc/make.texi +++ b/doc/make.texi @@ -4057,12 +4057,13 @@ If there is nothing looking like an integer after the @samp{-j} option, there is no limit on the number of job slots. The default number of job slots is one, which means serial execution (one thing at a time). -One consequence of running several recipes simultaneously is that by -default, output from each recipe appears as soon as it is generated, -with the result that messages from different recipes may be interspersed. -This may create problems in interpreting output. If the @samp{-P} option -is used, however, recipes will save their output until completion and -then take turns writing it, with a more coherent result. +When running several recipes simultaneously the output from each +recipe appears as soon as it is generated, with the result that +messages from different recipes may be interspersed. To avoid this +you can use the @samp{--output-sync} (@samp{-O}) option; if this +option is provided then the output from each recipe will be saved +until the recipe is complete, then written all at once. This ensures +that output from different recipes is not mixed together. Another problem is that two processes cannot both take input from the same device; so to make sure that only one recipe tries to take input @@ -8615,22 +8616,25 @@ The data base output contains file name and line number information for recipe and variable definitions, so it can be a useful debugging tool in complex environments. -@item -P -@cindex @code{-P} -@itemx --parallel-sync -@cindex @code{--parallel-sync} -@cindex parallel recipe execution, output -When jobs are running in parallel under @samp{--jobs}, the output of -each job is held until the job is complete thus ensuring that the output -of each recipe is grouped together. +@item -O +@cindex @code{-O} +@itemx --output-sync +@cindex @code{--output-sync} +@cindex output of parallel execution +@cindex parallel execution, output of +Ensure that the complete output from each recipe is printed in one +uninterrupted sequence. This option is only useful when using the +@code{--jobs} option to run multiple recipes simultaneously +(@pxref{Parallel, ,Parallel Execution}). Without this option output +will be displayed as it is generated by the recipes. With no argument or the argument @samp{1}, messages from each job in recursive makes are grouped together. With the argument @samp{2}, the complete output from any recursive make is grouped together. The latter achieves better grouping of output from related jobs, but causes longer delay, since messages do not appear until the recursive make has -completed. Therefore @samp{-P} is more useful when watching the output -while make runs, and @samp{-P2} is better suited when running a complex +completed. Therefore @samp{-O} is more useful when watching the output +while make runs, and @samp{-O2} is better suited when running a complex parallel build in the background and checking its output afterwards. @item -q @@ -243,15 +243,15 @@ unsigned long job_counter = 0; unsigned int jobserver_tokens = 0; -#ifdef PARALLEL_SYNC -/* Semaphore for use in -j mode with parallel_sync. */ +#ifdef OUTPUT_SYNC +/* Semaphore for use in -j mode with output_sync. */ int sync_handle = -1; -#define STREAM_OK(strm) ((fcntl (fileno ((strm)), F_GETFD) != -1) || (errno != EBADF)) +#define STREAM_OK(_s) ((fcntl (fileno (_s), F_GETFD) != -1) || (errno != EBADF)) -#define FD_NOT_EMPTY(FD) ((FD) >= 0 && lseek ((FD), 0, SEEK_CUR) > 0) -#endif /* PARALLEL_SYNC */ +#define FD_NOT_EMPTY(_f) ((_f) >= 0 && lseek ((_f), 0, SEEK_CUR) > 0) +#endif /* OUTPUT_SYNC */ #ifdef WINDOWS32 /* @@ -549,8 +549,8 @@ child_handler (int sig UNUSED) */ } -#ifdef PARALLEL_SYNC -/* Adds file descriptors to the child structure to support parallel_sync; +#ifdef OUTPUT_SYNC +/* Adds file descriptors to the child structure to support output_sync; one for stdout and one for stderr as long as they are (a) open and (b) separate. If stdout and stderr share a device they can share a temp file too. */ @@ -559,38 +559,38 @@ assign_child_tempfiles (struct child *c, int combined) { FILE *outstrm = NULL, *errstrm = NULL; int o_ok, e_ok; - const char *suppressed = "parallel-sync suppressed: "; + const char *suppressed = "output-sync suppressed: "; char *failmode = NULL; /* Check status of stdout and stderr before hooking up temp files. */ o_ok = STREAM_OK (stdout); e_ok = STREAM_OK (stderr); - /* The tmpfile() function returns a FILE pointer but those can be in + /* The tmpfile() function returns a FILE pointer but those can be in limited supply, so we immediately dup its file descriptor and keep only that, closing the FILE pointer. */ if (combined) { if (!(outstrm = tmpfile ())) - failmode = "tmpfile()"; + failmode = "tmpfile()"; else - errstrm = outstrm; + errstrm = outstrm; } else if (o_ok && e_ok) { if (!(outstrm = tmpfile ()) || !(errstrm = tmpfile ())) - failmode = "tmpfile()"; + failmode = "tmpfile()"; } else if (o_ok) { if (!(outstrm = tmpfile ())) - failmode = "tmpfile()"; + failmode = "tmpfile()"; } else if (e_ok) { if (!(errstrm = tmpfile ())) - failmode = "tmpfile()"; + failmode = "tmpfile()"; } else failmode = "stdout"; @@ -598,22 +598,22 @@ assign_child_tempfiles (struct child *c, int combined) if (!failmode && outstrm) { if ((c->outfd = dup (fileno (outstrm))) == -1) - failmode = "dup2()"; + failmode = "dup2()"; else - CLOSE_ON_EXEC (c->outfd); + CLOSE_ON_EXEC (c->outfd); } if (!failmode && errstrm) { if (combined) - c->errfd = c->outfd; + c->errfd = c->outfd; else - { - if ((c->errfd = dup (fileno (errstrm))) == -1) - failmode = "dup2()"; - else - CLOSE_ON_EXEC (c->errfd); - } + { + if ((c->errfd = dup (fileno (errstrm))) == -1) + failmode = "dup2()"; + else + CLOSE_ON_EXEC (c->errfd); + } } if (failmode) @@ -643,9 +643,9 @@ pump_from_tmp_fd (int from_fd, int to_fd) char *write_buf = buffer; EINTRLOOP (nleft, read (from_fd, buffer, sizeof (buffer))); if (nleft < 0) - perror ("read()"); + perror ("read()"); if (nleft <= 0) - break; + break; while (nleft > 0) { EINTRLOOP (nwrite, write (to_fd, write_buf, nleft)); @@ -702,10 +702,8 @@ sync_output (struct child *c) if ((outfd_not_empty || errfd_not_empty) && (sem = acquire_semaphore ())) { - /* - * We've entered the "critical section" during which a lock is held. - * We want to keep it as short as possible. - */ + /* We've entered the "critical section" during which a lock is held. + We want to keep it as short as possible. */ if (outfd_not_empty) { log_working_directory (1, 1); @@ -715,7 +713,7 @@ sync_output (struct child *c) if (errfd_not_empty && c->errfd != c->outfd) pump_from_tmp_fd (c->errfd, fileno (stderr)); - /* Exit the critical section */ + /* Exit the critical section. */ release_semaphore (sem); } @@ -725,7 +723,7 @@ sync_output (struct child *c) close (c->errfd); c->outfd = c->errfd = -1; } -#endif /* PARALLEL_SYNC */ +#endif /* OUTPUT_SYNC */ extern int shell_function_pid, shell_function_completed; @@ -1021,11 +1019,11 @@ reap_children (int block, int err) c->sh_batch_file = NULL; } -#ifdef PARALLEL_SYNC +#ifdef OUTPUT_SYNC /* Synchronize parallel output if requested */ - if (parallel_sync) - sync_output (c); -#endif /* PARALLEL_SYNC */ + if (output_sync) + sync_output (c); +#endif /* OUTPUT_SYNC */ /* If this child had the good stdin, say it is now free. */ if (c->good_stdin) @@ -1607,44 +1605,44 @@ start_job_command (struct child *child) #else /* !__EMX__ */ -#ifdef PARALLEL_SYNC - if (parallel_sync) - { - static int combined_output; - /* If parallel_sync is turned on, find a resource to - synchronize on. This block is traversed only once. */ - if (sync_handle == -1) - { - struct stat stbuf_o, stbuf_e; +#ifdef OUTPUT_SYNC + if (output_sync) + { + static int combined_output; + /* If output_sync is turned on, find a resource to + synchronize on. This block is traversed only once. */ + if (sync_handle == -1) + { + struct stat stbuf_o, stbuf_e; - if (STREAM_OK (stdout)) - { - sync_handle = fileno (stdout); - combined_output = - fstat (fileno (stdout), &stbuf_o) == 0 && - fstat (fileno (stderr), &stbuf_e) == 0 && - stbuf_o.st_dev == stbuf_e.st_dev && - stbuf_o.st_ino == stbuf_e.st_ino; - } - else if (STREAM_OK (stderr)) - sync_handle = fileno (stderr); - else - { - perror_with_name ("parallel-sync suppressed: ", "stderr"); - parallel_sync = 0; - } - } + if (STREAM_OK (stdout)) + { + sync_handle = fileno (stdout); + combined_output = + fstat (fileno (stdout), &stbuf_o) == 0 && + fstat (fileno (stderr), &stbuf_e) == 0 && + stbuf_o.st_dev == stbuf_e.st_dev && + stbuf_o.st_ino == stbuf_e.st_ino; + } + else if (STREAM_OK (stderr)) + sync_handle = fileno (stderr); + else + { + perror_with_name ("output-sync suppressed: ", "stderr"); + output_sync = 0; + } + } - /* If it still looks like we can synchronize, create a temp - file to hold stdout (and one for stderr if separate). */ - if (parallel_sync >= PARALLEL_SYNC_COARSE - || (parallel_sync == PARALLEL_SYNC_FINE && !(flags & COMMANDS_RECURSE))) - { - if (!assign_child_tempfiles (child, combined_output)) - parallel_sync = 0; - } - } -#endif /* PARALLEL_SYNC */ + /* If it still looks like we can synchronize, create a temp + file to hold stdout (and one for stderr if separate). */ + if (output_sync >= OUTPUT_SYNC_COARSE + || (output_sync == OUTPUT_SYNC_FINE && !(flags & COMMANDS_RECURSE))) + { + if (!assign_child_tempfiles (child, combined_output)) + output_sync = 0; + } + } +#endif /* OUTPUT_SYNC */ child->pid = vfork (); environ = parent_environ; /* Restore value child may have clobbered. */ @@ -1669,22 +1667,22 @@ start_job_command (struct child *child) setrlimit (RLIMIT_STACK, &stack_limit); #endif -#ifdef PARALLEL_SYNC - /* Divert child output into tempfile(s) if parallel_sync in use. */ - if (parallel_sync) - { - int outfd = fileno (stdout); - int errfd = fileno (stderr); +#ifdef OUTPUT_SYNC + /* Divert child output into tempfile(s) if output_sync in use. */ + if (output_sync) + { + int outfd = fileno (stdout); + int errfd = fileno (stderr); - if ((child->outfd >= 0 && - (close (outfd) == -1 || dup2 (child->outfd, outfd) == -1)) - || (child->errfd >= 0 && - (close (errfd) == -1 || dup2 (child->errfd, errfd) == -1))) - { - perror_with_name ("parallel-sync: ", "dup2()"); - } - } -#endif /* PARALLEL_SYNC */ + if ((child->outfd >= 0 && + (close (outfd) == -1 || dup2 (child->outfd, outfd) == -1)) + || (child->errfd >= 0 && + (close (errfd) == -1 || dup2 (child->errfd, errfd) == -1))) + { + perror_with_name ("output-sync: ", "dup2()"); + } + } +#endif /* OUTPUT_SYNC */ child_execute_job (child->good_stdin ? 0 : bad_stdin, 1, argv, child->environment); @@ -2019,7 +2017,7 @@ new_job (struct file *file) c->file = file; c->command_lines = lines; c->sh_batch_file = NULL; -#ifdef PARALLEL_SYNC +#ifdef OUTPUT_SYNC c->outfd = c->errfd = -1; #endif @@ -34,9 +34,9 @@ this program. If not, see <http://www.gnu.org/licenses/>. */ # define CLOSE_ON_EXEC(_d) (void) fcntl ((_d), F_SETFD, FD_CLOEXEC) #endif -#ifdef POSIX /* PARALLEL_SYNC */ -#define PARALLEL_SYNC -#endif /* POSIX */ +#ifdef POSIX +# define OUTPUT_SYNC +#endif /* Structure describing a running or dead child process. */ @@ -64,10 +64,10 @@ struct child unsigned int good_stdin:1; /* Nonzero if this child has a good stdin. */ unsigned int deleted:1; /* Nonzero if targets have been deleted. */ unsigned int dontcare:1; /* Saved dontcare flag. */ -#ifdef PARALLEL_SYNC - int outfd; /* Optional file descriptor for saving stdout */ - int errfd; /* Optional file descriptor for saving stderr */ -#endif /* PARALLEL_SYNC */ +#ifdef OUTPUT_SYNC + int outfd; /* File descriptor for saving stdout */ + int errfd; /* File descriptor for saving stderr */ +#endif }; extern struct child *children; @@ -228,12 +228,12 @@ static unsigned int master_job_slots = 0; static unsigned int inf_jobs = 0; -#ifdef PARALLEL_SYNC +#ifdef OUTPUT_SYNC -/* Default value for parallel sync without an argument. */ +/* Default value for output-sync without an argument. */ -static unsigned int no_parallel_sync = 0; -static unsigned int default_parallel_sync = PARALLEL_SYNC_FINE; +static unsigned int no_output_sync = 0; +static unsigned int default_output_sync = OUTPUT_SYNC_FINE; #endif @@ -351,13 +351,13 @@ static const char *const usage[] = N_("\ -o FILE, --old-file=FILE, --assume-old=FILE\n\ Consider FILE to be very old and don't remake it.\n"), +#ifdef OUTPUT_SYNC N_("\ - -p, --print-data-base Print make's internal database.\n"), -#ifdef PARALLEL_SYNC - N_("\ - -P [2], --parallel-sync[=2] Synchronize output of parallel jobs [coarse].\n"), + -O [2], --output-sync[=2] Synchronize output of parallel jobs [coarse].\n"), #endif N_("\ + -p, --print-data-base Print make's internal database.\n"), + N_("\ -q, --question Run no recipe; exit status says if up to date.\n"), N_("\ -r, --no-builtin-rules Disable the built-in implicit rules.\n"), @@ -420,12 +420,12 @@ static const struct command_switch switches[] = { 'm', ignore, 0, 0, 0, 0, 0, 0, 0 }, { 'n', flag, &just_print_flag, 1, 1, 1, 0, 0, "just-print" }, { 'o', filename, &old_files, 0, 0, 0, 0, 0, "old-file" }, - { 'p', flag, &print_data_base_flag, 1, 1, 0, 0, 0, "print-data-base" }, -#ifdef PARALLEL_SYNC - // { 'P', flag, ¶llel_sync, 1, 1, 0, 0, 0, "parallel-sync" }, // two-state - { 'P', positive_int, ¶llel_sync, 1, 1, 0, &default_parallel_sync, - &no_parallel_sync, "parallel-sync" }, +#ifdef OUTPUT_SYNC + // { 'O', flag, &output_sync, 1, 1, 0, 0, 0, "output-sync" }, // two-state + { 'O', positive_int, &output_sync, 1, 1, 0, &default_output_sync, + &no_output_sync, "output-sync" }, #endif + { 'p', flag, &print_data_base_flag, 1, 1, 0, 0, 0, "print-data-base" }, { 'q', flag, &question_flag, 1, 1, 1, 0, 0, "question" }, { 'r', flag, &no_builtin_rules_flag, 1, 1, 0, 0, 0, "no-builtin-rules" }, { 'R', flag, &no_builtin_variables_flag, 1, 1, 0, 0, 0, @@ -521,13 +521,13 @@ int second_expansion; int one_shell; -/* Either PARALLEL_SYNC_FINE or PARALLEL_SYNC_COARSE - if the "--parallel-sync" option was given. +/* Either OUTPUT_SYNC_FINE or OUTPUT_SYNC_COARSE + if the "--output-sync" option was given. This attempts to synchronize the output of parallel jobs such that the results of each job stay together. It works best in combination with .ONESHELL. */ -int parallel_sync; +int output_sync; /* Nonzero if we have seen the '.NOTPARALLEL' target. This turns off parallel builds for this invocation of make. */ @@ -1763,7 +1763,7 @@ main (int argc, char **argv, char **envp) if (! open_jobserver_semaphore(cp)) { DWORD err = GetLastError(); - fatal (NILF, _("internal error: unable to open jobserver semaphore '%s': (Error %ld: %s)"), + fatal (NILF, _("internal error: unable to open jobserver semaphore '%s': (Error %ld: %s)"), cp, err, map_windows32_error_to_string(err)); } DB (DB_JOBS, (_("Jobserver client (semaphore %s)\n"), cp)); @@ -525,8 +525,8 @@ int strncasecmp (const char *s1, const char *s2, int n); # endif #endif -#define PARALLEL_SYNC_FINE 1 -#define PARALLEL_SYNC_COARSE 2 +#define OUTPUT_SYNC_FINE 1 +#define OUTPUT_SYNC_COARSE 2 extern const gmk_floc *reading_file; extern const gmk_floc **expanding_var; @@ -539,7 +539,7 @@ extern int env_overrides, no_builtin_rules_flag, no_builtin_variables_flag; extern int print_version_flag, print_directory_flag, check_symlink_flag; extern int warn_undefined_variables_flag, trace_flag, posix_pedantic; extern int not_parallel, second_expansion, clock_skew_detected; -extern int rebuilding_makefiles, one_shell, parallel_sync; +extern int rebuilding_makefiles, one_shell, output_sync; /* can we run commands via 'sh -c xxx' or must we use batch files? */ extern int batch_mode_shell; @@ -235,7 +235,7 @@ message (prefix, fmt, va_alist) if (fmt != 0) { - if (parallel_sync) + if (output_sync) log_working_directory (1, 1); if (prefix) @@ -250,7 +250,7 @@ message (prefix, fmt, va_alist) VA_END (args); putchar ('\n'); - if (parallel_sync) + if (output_sync) log_working_directory (0, 1); } @@ -273,7 +273,7 @@ error (flocp, fmt, va_alist) va_list args; #endif - if (parallel_sync) + if (output_sync) log_working_directory (1, 1); else log_working_directory (1, 0); @@ -292,7 +292,7 @@ error (flocp, fmt, va_alist) putc ('\n', stderr); fflush (stderr); - if (parallel_sync) + if (output_sync) log_working_directory (0, 1); } @@ -312,7 +312,7 @@ fatal (flocp, fmt, va_alist) va_list args; #endif - if (parallel_sync) + if (output_sync) log_working_directory (1, 1); else log_working_directory (1, 0); @@ -330,7 +330,7 @@ fatal (flocp, fmt, va_alist) fputs (_(". Stop.\n"), stderr); - if (parallel_sync) + if (output_sync) log_working_directory (0, 1); die (2); diff --git a/tests/scripts/features/parallel-sync b/tests/scripts/features/output-sync index 29e5564..66c0936 100644 --- a/tests/scripts/features/parallel-sync +++ b/tests/scripts/features/output-sync @@ -1,6 +1,6 @@ # -*-perl-*- -$description = "Test parallel-sync (-P) option."; +$description = "Test --output-sync (-O) option."; $details = "Test the synchronization of output from parallel jobs."; @@ -24,11 +24,9 @@ open(MAKEFILE,"> foo/Makefile"); print MAKEFILE <<EOF; all: foo -foo: - \@echo foo: start; $sleep_command 2; echo foo: end +foo: ; \@echo foo: start; $sleep_command 2; echo foo: end -foo-fail: - \@$sleep_command 2; false +foo-fail: ; \@$sleep_command 2; false EOF close(MAKEFILE); @@ -36,11 +34,9 @@ open(MAKEFILE,"> bar/Makefile"); print MAKEFILE <<EOF; all: bar baz -bar: - \@echo bar: start; $sleep_command 1; echo bar: end +bar: ; \@echo bar: start; $sleep_command 1; echo bar: end -baz: - \@echo baz: start; $sleep_command 4; echo baz: end +baz: ; \@echo baz: start; $sleep_command 4; echo baz: end EOF close(MAKEFILE); @@ -48,12 +44,10 @@ close(MAKEFILE); run_make_test(' all: make-foo make-bar -make-foo: - $(MAKE) -C foo +make-foo: ; $(MAKE) -C foo -make-bar: - $(MAKE) -C bar', -'-j -P2', +make-bar: ; $(MAKE) -C bar', +'-j -O2', "#MAKEPATH# -C foo #MAKEPATH# -C bar #MAKE#[1]: Entering directory '#PWD#/foo' @@ -77,12 +71,10 @@ baz: end run_make_test(' all: make-foo make-bar -make-foo: - $(MAKE) -C foo +make-foo: ; $(MAKE) -C foo -make-bar: - $(MAKE) -C bar', -'-j -P', +make-bar: ; $(MAKE) -C bar', +'-j --output-sync', "#MAKEPATH# -C foo #MAKEPATH# -C bar #MAKE#[1]: Entering directory '#PWD#/foo' @@ -107,12 +99,10 @@ baz: end run_make_test(' all: make-foo-fail make-bar-bar -make-foo-fail: - $(MAKE) -C foo foo-fail +make-foo-fail: ; $(MAKE) -C foo foo-fail -make-bar-bar: - $(MAKE) -C bar bar', -'-j -P', +make-bar-bar: ; $(MAKE) -C bar bar', +'-j -O', "#MAKEPATH# -C foo foo-fail #MAKEPATH# -C bar bar #MAKE#[1]: Entering directory '#PWD#/foo' @@ -123,13 +113,13 @@ bar: end #MAKE#[1]: Leaving directory '#PWD#/bar' #MAKE#[1]: Leaving directory '#PWD#/bar' #MAKE#[1]: Entering directory '#PWD#/foo' -Makefile:7: recipe for target 'foo-fail' failed +Makefile:5: recipe for target 'foo-fail' failed #MAKE#[1]: Leaving directory '#PWD#/foo' #MAKE#[1]: Entering directory '#PWD#/foo' #MAKE#[1]: *** [foo-fail] Error 1 #MAKE#[1]: Leaving directory '#PWD#/foo' #MAKE#[1]: Leaving directory '#PWD#/foo' -#MAKEFILE#:5: recipe for target 'make-foo-fail' failed +#MAKEFILE#:4: recipe for target 'make-foo-fail' failed #MAKE#: *** [make-foo-fail] Error 2\n", 512); |