use C4::Context;
use Getopt::Long;
+use Fcntl qw(:flock);
use File::Temp qw/ tempdir /;
use File::Path;
use C4::Biblio;
use Koha::RecordProcessor;
use XML::LibXML;
+use constant LOCK_FILENAME => 'rebuild..LCK';
+
# script that checks zebradir structure & create directories & mandatory files if needed
#
#
my $want_help;
my $as_xml;
my $process_zebraqueue;
+my $process_zebraqueue_skip_deletes;
my $do_not_clear_zebraqueue;
my $length;
my $where;
my $offset;
my $run_as_root;
my $run_user = (getpwuid($<))[0];
+my $wait_for_lock = 0;
+my $use_flock;
my $verbose_logging = 0;
my $zebraidx_log_opt = " -v none,fatal,warn ";
'x' => \$as_xml,
'y' => \$do_not_clear_zebraqueue,
'z' => \$process_zebraqueue,
- 'where:s' => \$where,
- 'length:i' => \$length,
+ 'skip-deletes' => \$process_zebraqueue_skip_deletes,
+ 'where:s' => \$where,
+ 'length:i' => \$length,
'offset:i' => \$offset,
- 'v+' => \$verbose_logging,
- 'run-as-root' => \$run_as_root,
+ 'v+' => \$verbose_logging,
+ 'run-as-root' => \$run_as_root,
+ 'wait-for-lock' => \$wait_for_lock,
);
if (not $result or $want_help) {
my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber","");
my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber","");
+# Protect again simultaneous update of the zebra index by using a lock file.
+# Create our own lock directory if its missing. This shouild be created
+# by koha-zebra-ctl.sh or at system installation. If the desired directory
+# does not exist and cannot be created, we fall back on /tmp - which will
+# always work.
+
+my ($lockfile, $LockFH);
+foreach (
+ C4::Context->config("zebra_lockdir"),
+ '/var/lock/zebra_' . C4::Context->config('database'),
+ '/tmp/zebra_' . C4::Context->config('database')
+) {
+ #we try three possibilities (we really want to lock :)
+ next if !$_;
+ ($LockFH, $lockfile) = _create_lockfile($_.'/rebuild');
+ last if defined $LockFH;
+}
+if( !defined $LockFH ) {
+ print "WARNING: Could not create lock file $lockfile: $!\n";
+ print "Please check your koha-conf.xml for ZEBRA_LOCKDIR.\n";
+ print "Verify file permissions for it too.\n";
+ $use_flock = 0; # we disable file locking now and will continue
+ # without it
+ # note that this mimics old behavior (before we used
+ # the lockfile)
+};
+
if ( $verbose_logging ) {
print "Zebra configuration information\n";
print "================================\n";
print "Zebra biblio directory = $biblioserverdir\n";
print "Zebra authorities directory = $authorityserverdir\n";
print "Koha directory = $kohadir\n";
+ print "Lockfile = $lockfile\n" if $lockfile;
print "BIBLIONUMBER in : $biblionumbertagfield\$$biblionumbertagsubfield\n";
print "BIBLIOITEMNUMBER in : $biblioitemnumbertagfield\$$biblioitemnumbertagsubfield\n";
print "================================\n";
my $tester = XML::LibXML->new();
+# The main work is done here by calling do_one_pass(). We have added locking
+# avoid race conditions between full rebuilds and incremental updates either from
+# daemon mode or periodic invocation from cron. The race can lead to an updated
+# record being overwritten by a rebuild if the update is applied after the export
+# by the rebuild and before the rebuild finishes (more likely to affect large
+# catalogs).
+#
+# We have chosen to exit immediately by default if we cannot obtain the lock
+# to prevent the potential for a infinite backlog from cron invocations, but an
+# option (wait-for-lock) is provided to let the program wait for the lock.
+# See http://bugs.koha-community.org/bugzilla3/show_bug.cgi?id=11078 for details.
if ($daemon_mode) {
while (1) {
- do_one_pass() if ( zebraqueue_not_empty() );
+ # For incremental updates, skip the update if the updates are locked
+ if (_flock($LockFH, LOCK_EX|LOCK_NB)) {
+ do_one_pass() if ( zebraqueue_not_empty() );
+ _flock($LockFH, LOCK_UN);
+ }
sleep $daemon_sleep;
}
} else {
- do_one_pass();
+ # all one-off invocations
+ my $lock_mode = ($wait_for_lock) ? LOCK_EX : LOCK_EX|LOCK_NB;
+ if (_flock($LockFH, $lock_mode)) {
+ do_one_pass();
+ _flock($LockFH, LOCK_UN);
+ } else {
+ print "Skipping rebuild/update because flock failed on $lockfile: $!\n";
+ }
}
$where_str = 'server = "authorityserver" AND done = 0;';
}
my $query =
- $dbh->prepare( 'SELECT COUNT(*) FROM zebraqueue WHERE ' . $where_str );
+ $dbh->prepare('SELECT COUNT(*) FROM zebraqueue WHERE ' . $where_str );
$query->execute;
my $count = $query->fetchrow_arrayref->[0];
my ($record_type, $directory, $skip_export, $skip_index, $process_zebraqueue, $as_xml, $noxml, $nosanitize, $do_not_clear_zebraqueue, $verbose_logging, $zebraidx_log_opt, $server_dir) = @_;
my $num_records_exported = 0;
- my $records_deleted;
+ my $records_deleted = {};
my $need_reset = check_zebra_dirs($server_dir);
if ($need_reset) {
print "$0: found broken zebra server directories: forcing a rebuild\n";
mkdir "$directory" unless (-d $directory);
mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
if ($process_zebraqueue) {
- my $entries = select_zebraqueue_records($record_type, 'deleted');
- mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
- $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
- mark_zebraqueue_batch_done($entries);
+ my $entries;
+
+ unless ( $process_zebraqueue_skip_deletes ) {
+ $entries = select_zebraqueue_records($record_type, 'deleted');
+ mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
+ $records_deleted = generate_deleted_marc_records($record_type, $entries, "$directory/del_$record_type", $as_xml);
+ mark_zebraqueue_batch_done($entries);
+ }
+
$entries = select_zebraqueue_records($record_type, 'updated');
mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
- $num_records_exported = export_marc_records_from_list($record_type,
- $entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
+ $num_records_exported = export_marc_records_from_list($record_type,$entries, "$directory/upd_$record_type", $as_xml, $noxml, $records_deleted);
mark_zebraqueue_batch_done($entries);
+
} else {
my $sth = select_all_records($record_type);
$num_records_exported = export_marc_records_from_sth($record_type, $sth, "$directory/$record_type", $as_xml, $noxml, $nosanitize);
}
+sub _flock {
+ # test if flock is present; if so, use it; if not, return true
+ # op refers to the official flock operations including LOCK_EX,
+ # LOCK_UN, etc.
+ # combining LOCK_EX with LOCK_NB returns immediately
+ my ($fh, $op)= @_;
+ if( !defined($use_flock) ) {
+ #check if flock is present; if not, you will have a fatal error
+ my $lock_acquired = eval { flock($fh, $op) };
+ # assuming that $fh and $op are fine(..), an undef $lock_acquired
+ # means no flock
+ $use_flock = defined($lock_acquired) ? 1 : 0;
+ print "Warning: flock could not be used!\n" if $verbose_logging && !$use_flock;
+ return 1 if !$use_flock;
+ return $lock_acquired;
+ } else {
+ return 1 if !$use_flock;
+ return flock($fh, $op);
+ }
+}
+
+sub _create_lockfile { #returns undef on failure
+ my $dir= shift;
+ unless (-d $dir) {
+ eval { mkpath($dir, 0, oct(755)) };
+ return if $@;
+ }
+ return if !open my $fh, q{>}, $dir.'/'.LOCK_FILENAME;
+ return ( $fh, $dir.'/'.LOCK_FILENAME );
+}
+
sub print_usage {
print <<_USAGE_;
$0: reindex MARC bibs and/or authorities in Zebra.
table. Cannot be used with -r
or -s.
+ --skip-deletes only select record updates, not record
+ deletions, to avoid potential excessive
+ I/O when zebraidx processes deletions.
+ If this option is used for normal indexing,
+ a cronjob should be set up to run
+ rebuild_zebra.pl -z without --skip-deletes
+ during off hours.
+ Only effective with -z.
+
-r clear Zebra index before
adding records to index. Implies -w.
--run-as-root explicitily allow script to run as 'root' user
+ --wait-for-lock when not running in daemon mode, the default
+ behavior is to abort a rebuild if the rebuild
+ lock is busy. This option will cause the program
+ to wait for the lock to free and then continue
+ processing the rebuild request,
+
--help or -h show this message.
_USAGE_
}
projects / koha_fer / blobdiff