Bug 32558: Add ability for background_jobs_worker.pl to process multiple jobs simulta...

Right now background_jobs_worker.pl only processes jobs in serial. It would make sense to handle jobs in parallel up to a user definable limit.

Test Plan:
1) Apply this patch
2) Stop background_jobs_worker.pl
3) Generate some background jobs by editing records, placing holds, etc
4) Watch processes in a new terminal: watch -n 0.1 'ps aux | grep background_jobs_worker.pl'
5) Run background_jobs_worker.pl with parameter -m 3 or some other
   number of max processes
6) Note the multiple forked processes in the ps output

Test notes - also tested the following on KTD:
1. Stop background_jobs_worker.pl
2. Edit /etc/koha/sites/kohadev/koha-conf.xml - set max_processes to 10
3. Generate some background jobs
4. Watch processes in a new terminal: watch -n 0.1 'ps aux | grep background_jobs_worker.pl'
5. Restart all
6. Confirm multiple forked processes in the ps output

Both methods work as expected and generate multiple forked processes
based on the value set for max processes.

Signed-off-by: emlam <emily.lamancusa@montgomerycountymd.gov>
Signed-off-by: Marcel de Rooy <m.de.rooy@rijksmuseum.nl>
Signed-off-by: Tomas Cohen Arazi <tomascohen@theke.io>

diff --git a/debian/templates/koha-conf-site.xml.in b/debian/templates/koha-conf-site.xml.in
index bdb994c..187150f 100644 (file)
--- a/debian/templates/koha-conf-site.xml.in
+++ b/debian/templates/koha-conf-site.xml.in
@@ -457,6 +457,11 @@ __END_SRU_PUBLICSERVER__
    <vhost>__MESSAGE_BROKER_VHOST__</vhost>
  </message_broker>
 
+ <background_jobs_worker>
+     <!-- Max simultaneous processes per worker -->
+     <max_processes>1</max_processes>
+ </background_jobs_worker>
+
  <do_not_remove_cookie>__KEEP_COOKIE__</do_not_remove_cookie>
  <do_not_remove_cookie>catalogue_editor_\d+</do_not_remove_cookie>
  <!-- Uncomment lines like hereunder to not clear cookies at logout:

diff --git a/etc/koha-conf.xml b/etc/koha-conf.xml
index 3e5623c..beb5ccd 100644 (file)
--- a/etc/koha-conf.xml
+++ b/etc/koha-conf.xml
@@ -274,6 +274,11 @@
    <vhost></vhost>
  </message_broker>
 
+ <background_jobs_worker>
+     <!-- Max simultaneous processes per worker -->
+     <max_processes>1</max_processes>
+ </background_jobs_worker>
+
  <do_not_remove_cookie>catalogue_editor_\d+</do_not_remove_cookie>
  <!-- Uncomment lines like hereunder to not clear cookies at logout:
       The cookie name is case sensitive.

diff --git a/misc/workers/background_jobs_worker.pl b/misc/workers/background_jobs_worker.pl
index 59920a0..c5b84ad 100755 (executable)
--- a/misc/workers/background_jobs_worker.pl
+++ b/misc/workers/background_jobs_worker.pl
@@ -21,7 +21,7 @@ background_jobs_worker.pl - Worker script that will process background jobs
 
 =head1 SYNOPSIS
 
-./background_jobs_worker.pl [--queue QUEUE]
+./background_jobs_worker.pl [--queue QUEUE] [-m|--max-processes MAX_PROCESSES]
 
 =head1 DESCRIPTION
 
@@ -30,6 +30,11 @@ or if a Stomp server is not active it will poll the database every 10s for new j
 
 You can specify some queues only (using --queue, which is repeatable) if you want to run several workers that will handle their own jobs.
 
+--m --max-processes specifies how many jobs to process simultaneously
+
+Max processes will be set from the command line option, the environment variable MAX_PROCESSES, or the koha-conf file, in that order of precedence.
+By default the script will only run one job at a time.
+
 =head1 OPTIONS
 
 =over
@@ -52,17 +57,26 @@ use JSON qw( decode_json );
 use Try::Tiny;
 use Pod::Usage;
 use Getopt::Long;
+use Parallel::ForkManager;
 
 use C4::Context;
 use Koha::Logger;
 use Koha::BackgroundJobs;
+use C4::Context;
 
 my ( $help, @queues );
+
+my $max_processes = $ENV{MAX_PROCESSES};
+$max_processes ||= C4::Context->config('background_jobs_worker')->{max_processes} if C4::Context->config('background_jobs_worker');
+$max_processes ||= 1;
+
 GetOptions(
+    'm|max-processes=i' => \$max_processes,
     'h|help' => \$help,
     'queue=s' => \@queues,
 ) || pod2usage(1);
 
+
 pod2usage(0) if $help;
 
 unless (@queues) {
@@ -76,6 +90,8 @@ try {
     warn sprintf "Cannot connect to the message broker, the jobs will be processed anyway (%s)", $_;
 };
 
+my $pm = Parallel::ForkManager->new($max_processes);
+
 if ( $conn ) {
     # FIXME cf note in Koha::BackgroundJob about $namespace
     my $namespace = C4::Context->config('memcached_namespace');
@@ -118,7 +134,9 @@ while (1) {
             next;
         }
 
+        $pm->start and next;
         process_job( $job, $args );
+        $pm->finish;
 
     } else {
         my $jobs = Koha::BackgroundJobs->search({ status => 'new', queue => \@queues });
@@ -133,7 +151,9 @@ while (1) {
 
             next unless $args;
 
+            $pm->start and next;
             process_job( $job, { job_id => $job->id, %$args } );
+            $pm->finish;
 
         }
         sleep 10;