$|=1; # flushes output
-# limit for database dumping
my $directory;
my $skip_export;
my $keep_export;
my $do_munge;
my $want_help;
my $as_xml;
+my $process_zebraqueue;
my $result = GetOptions(
'd:s' => \$directory,
'reset' => \$reset,
'a' => \$authorities,
'h|help' => \$want_help,
'x' => \$as_xml,
+ 'z' => \$process_zebraqueue,
);
}
if (not $biblios and not $authorities) {
- my $msg = "Must specify -b or -a to reindex bibs or authorites\n";
+ my $msg = "Must specify -b or -a to reindex bibs or authorities\n";
+ $msg .= "Please do '$0 --help' to see usage.\n";
+ die $msg;
+}
+
+if ($authorities and $as_xml) {
+ my $msg = "Cannot specify both -a and -x\n";
+ $msg .= "Please do '$0 --help' to see usage.\n";
+ die $msg;
+}
+
+if ($process_zebraqueue and ($skip_export or $reset)) {
+ my $msg = "Cannot specify -r or -s if -z is specified\n";
$msg .= "Please do '$0 --help' to see usage.\n";
die $msg;
}
munge_config();
}
+$dbh->{AutoCommit} = 0; # don't autocommit - want a consistent view of the zebraqueue table
+
if ($authorities) {
- #
- # exporting authorities
- #
- if ($skip_export) {
- print "====================\n";
- print "SKIPPING authorities export\n";
- print "====================\n";
- } else {
- print "====================\n";
- print "exporting authorities\n";
- print "====================\n";
- mkdir "$directory" unless (-d $directory);
- mkdir "$directory/authorities" unless (-d "$directory/authorities");
- open(OUT,">:utf8","$directory/authorities/authorities.iso2709") or die $!;
- my $dbh=C4::Context->dbh;
- my $sth;
- $sth=$dbh->prepare("select authid,marc from auth_header");
- $sth->execute();
- my $i=0;
- while (my ($authid,$record) = $sth->fetchrow) {
- # FIXME : we retrieve the iso2709 record. if the GetAuthority (that uses the XML) fails
- # due to some MARC::File::XML failure, then try the iso2709,
- # (add authid & authtype if needed)
- my $record;
- eval {
- $record = GetAuthority($authid);
- };
- next unless $record;
- # force authid in case it's not here, otherwise, zebra will die on this authority
- unless ($record->field('001')->data() eq $authid){
- print "$authid don't exist for this authority :".$record->as_formatted;
- $record->delete_field($record->field('001'));
- $record->insert_fields_ordered(MARC::Field->new('001',$authid));
- }
- if($@){
- print " There was some pb getting authority : ".$authid."\n";
- next;
- }
-
- print ".";
- print "\r$i" unless ($i++ %100);
-# # remove leader length, that could be wrong, it will be calculated automatically by as_usmarc
-# # otherwise, if it's wron, zebra will fail miserabily (and never index what is after the failing record)
- my $leader=$record->leader;
- substr($leader,0,5)=' ';
- substr($leader,10,7)='22 ';
- $record->leader(substr($leader,0,24));
- print OUT $record->as_usmarc;
- }
- close(OUT);
- }
-
- #
- # and reindexing everything
- #
- print "====================\n";
- print "REINDEXING zebra\n";
- print "====================\n";
- do_indexing('authority', 'update', "$directory/authorities", $reset, $noshadow, 'iso2709');
+ index_records('authority', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml);
+ $dbh->commit(); # commit changes to zebraqueue, if any
} else {
print "skipping authorities\n";
}
-#################################################################################################################
-# BIBLIOS
-#################################################################################################################
if ($biblios) {
- # die;
- #
- # exporting biblios
- #
- if ($skip_export) {
- print "====================\n";
- print "SKIPPING biblio export\n";
- print "====================\n";
- } else {
- print "====================\n";
- print "exporting biblios\n";
- print "====================\n";
- mkdir "$directory" unless (-d $directory);
- mkdir "$directory/biblios" unless (-d "$directory/biblios");
- open(OUT,">:utf8 ","$directory/biblios/export") or die $!;
- my $dbh=C4::Context->dbh;
- my $sth;
- if ($noxml){
- $sth=$dbh->prepare("select biblionumber,marc from biblioitems order by biblionumber");
- $sth->execute();
- my $i=0;
- while (my ($biblionumber,$marc) = $sth->fetchrow) {
- my $record;
- $record=MARC::Record->new_from_usmarc($marc);
- my $record_correct=1;
- # skip uncorrect records : isn't this bogus, as just after we reintroduce biblionumber if it's missing ?
- # FIXME next unless $record->field($biblionumbertagfield);
- # check if biblionumber is present, otherwise, add it on the fly
- if ($biblionumbertagfield eq '001') {
- unless ($record->field($biblionumbertagfield)->data()) {
- $record_correct=0;
- my $field;
- # if the field where biblionumber is already exist, just update it, otherwise create it
- if ($record->field($biblionumbertagfield)) {
- $field = $record->field($biblionumbertagfield);
- $field->update($biblionumber);
- } else {
- my $newfield;
- $newfield = MARC::Field->new( $biblionumbertagfield, $biblionumber);
- $record->append_fields($newfield);
- }
- }
- } else {
- unless ($record->subfield($biblionumbertagfield,$biblionumbertagsubfield)) {
- $record_correct=0;
- my $field;
- # if the field where biblionumber is already exist, just update it, otherwise create it
- if ($record->field($biblionumbertagfield)) {
- $field = $record->field($biblionumbertagfield);
- $field->add_subfields($biblionumbertagsubfield => $biblionumber);
- } else {
- my $newfield;
- $newfield = MARC::Field->new( $biblionumbertagfield,'','', $biblionumbertagsubfield => $biblionumber);
- $record->append_fields($newfield);
- }
- }
- # warn "FIXED BIBLIONUMBER".$record->as_formatted;
- }
- unless ($record->subfield($biblioitemnumbertagfield,$biblioitemnumbertagsubfield)) {
- $record_correct=0;
- # warn "INCORRECT BIBLIOITEMNUMBER :".$record->as_formatted;
- my $field;
- # if the field where biblionumber is already exist, just update it, otherwise create it
- if ($record->field($biblioitemnumbertagfield)) {
- $field = $record->field($biblioitemnumbertagfield);
- if ($biblioitemnumbertagfield <10) {
- $field->update($biblionumber);
- } else {
- $field->add_subfields($biblioitemnumbertagsubfield => $biblionumber);
- }
- } else {
- my $newfield;
- if ($biblioitemnumbertagfield <10) {
- $newfield = MARC::Field->new( $biblioitemnumbertagfield, $biblionumber);
- } else {
- $newfield = MARC::Field->new( $biblioitemnumbertagfield,'','', $biblioitemnumbertagsubfield => $biblionumber);
- }
- $record->insert_grouped_field($newfield);
- }
- # warn "FIXED BIBLIOITEMNUMBER".$record->as_formatted;
- }
- my $leader=$record->leader;
- substr($leader,0,5)=' ';
- substr($leader,10,7)='22 ';
- $record->leader(substr($leader,0,24));
- print OUT $record->as_usmarc();
- }
- close (OUT);
- } else {
- $sth=$dbh->prepare("SELECT biblionumber FROM biblioitems ORDER BY biblionumber");
- $sth->execute();
- my $i=0;
- while (my ($biblionumber) = $sth->fetchrow) {
- print ".";
- print "\r$i" unless ($i++ %100);
- my $record;
- eval {
- $record = GetMarcBiblio($biblionumber);
- };
- if($@){
- print " There was some pb getting biblio : #".$biblionumber."\n";
- next;
- }
- next unless $record;
-# die if $record->subfield('090','9') eq 11;
- # print $record;
- # check that biblionumber & biblioitemnumber are stored in the MARC record, otherwise, add them & update the biblioitems.marcxml data.
- my $record_correct=1;
- # skip uncorrect records : isn't this bogus, as just after we reintroduce biblionumber if it's missing ?
- # FIXME next unless $record->field($biblionumbertagfield);
- #
- #
- # CHECK biblionumber
- #
- #
- if ($biblionumbertagfield eq '001') {
- unless ($record->field($biblionumbertagfield) && $record->field($biblionumbertagfield)->data()) {
- $record_correct=0;
- my $field;
- # if the field where biblionumber is already exist, just update it, otherwise create it
- if ($record->field($biblionumbertagfield)) {
- $field = $record->field($biblionumbertagfield);
- $field->update($biblionumber);
- } else {
- my $newfield;
- $newfield = MARC::Field->new( $biblionumbertagfield, $biblionumber);
- $record->append_fields($newfield);
- }
- }
- } else {
- unless ($record->subfield($biblionumbertagfield,$biblionumbertagsubfield)) {
-# warn "fixing biblionumber for $biblionumbertagfield,$biblionumbertagsubfield = $biblionumber";
- $record_correct=0;
- my $field;
- # if the field where biblionumber is already exist, just update it, otherwise create it
- if ($record->field($biblionumbertagfield)) {
- $field = $record->field($biblionumbertagfield);
- $field->add_subfields($biblionumbertagsubfield => $biblionumber);
- } else {
- my $newfield;
- $newfield = MARC::Field->new( $biblionumbertagfield,'','', $biblionumbertagsubfield => $biblionumber);
- $record->append_fields($newfield);
- }
- }
-# warn "FIXED BIBLIONUMBER".$record->as_formatted;
- }
- #
- #
- # CHECK BIBLIOITEMNUMBER
- #
- #
- unless ($record->subfield($biblioitemnumbertagfield,$biblioitemnumbertagsubfield)) {
-# warn "fixing biblioitemnumber for $biblioitemnumbertagfield,$biblioitemnumbertagsubfield = $biblionumber";
- $record_correct=0;
- my $field;
- # if the field where biblionumber is already exist, just update it, otherwise create it
- if ($record->field($biblioitemnumbertagfield)) {
- $field = $record->field($biblioitemnumbertagfield);
- if ($biblioitemnumbertagfield <10) {
- $field->update($biblionumber);
- } else {
- $field->add_subfields($biblioitemnumbertagsubfield => $biblionumber);
- }
- } else {
- my $newfield;
- if ($biblioitemnumbertagfield <10) {
- $newfield = MARC::Field->new( $biblioitemnumbertagfield, $biblionumber);
- } else {
- $newfield = MARC::Field->new( $biblioitemnumbertagfield,'','', $biblioitemnumbertagsubfield => $biblionumber);
- }
- $record->insert_grouped_field($newfield);
- }
- # warn "FIXED BIBLIOITEMNUMBER".$record->as_formatted;
- }
- #
- #
- # CHECK FIELD 100
- #
- #
- my $encoding = C4::Context->preference("marcflavour");
- # deal with UNIMARC field 100 (encoding) : create it if needed & set encoding to unicode
- if ( $encoding eq "UNIMARC" ) {
- my $string;
- if ( length($record->subfield( 100, "a" )) == 35 ) {
- $string = $record->subfield( 100, "a" );
- my $f100 = $record->field(100);
- $record->delete_field($f100);
- }
- else {
- $string = POSIX::strftime( "%Y%m%d", localtime );
- $string =~ s/\-//g;
- $string = sprintf( "%-*s", 35, $string );
- }
- substr( $string, 22, 6, "frey50" );
- unless ( length($record->subfield( 100, "a" )) == 35 ) {
- $record->delete_field($record->field(100));
- $record->insert_grouped_field(
- MARC::Field->new( 100, "", "", "a" => $string ) );
- }
- }
- unless ($record_correct) {
- my $update_xml = $dbh->prepare("update biblioitems set marcxml=? where biblionumber=?");
- warn "UPDATING $biblionumber (missing biblionumber or biblioitemnumber in MARC record : ".$record->as_xml;
- $update_xml->execute($record->as_xml,$biblionumber);
- }
- # remove leader length, that could be wrong, it will be calculated automatically by as_usmarc
- # otherwise, if it's wron, zebra will fail miserabily (and never index what is after the failing record)
- my $leader=$record->leader;
- substr($leader,0,5)=' ';
- substr($leader,10,7)='22 ';
- $record->leader(substr($leader,0,24));
- if($as_xml) {
- print OUT $record->as_xml_record();
- } else {
- print OUT $record->as_usmarc();
- }
- }
- }
- close(OUT);
- }
-
- #
- # and reindexing everything
- #
- print "====================\n";
- print "REINDEXING zebra\n";
- print "====================\n";
- my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
- do_indexing('biblio', 'update', "$directory/biblios", $reset, $noshadow, $record_fmt);
+ index_records('biblio', $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml);
+ $dbh->commit(); # commit changes to zebraqueue, if any
} else {
print "skipping biblios\n";
}
+
print "====================\n";
print "CLEANING\n";
print "====================\n";
}
}
+sub index_records {
+ my ($record_type, $directory, $skip_export, $process_zebraqueue, $as_xml, $noxml) = @_;
+
+ my $num_records_exported = 0;
+ my $num_records_deleted = 0;
+ if ($skip_export) {
+ print "====================\n";
+ print "SKIPPING $record_type export\n";
+ print "====================\n";
+ } else {
+ print "====================\n";
+ print "exporting $record_type\n";
+ print "====================\n";
+ mkdir "$directory" unless (-d $directory);
+ mkdir "$directory/$record_type" unless (-d "$directory/$record_type");
+ if ($process_zebraqueue) {
+ my $sth = select_zebraqueue_records($record_type, 'deleted');
+ mkdir "$directory/del_$record_type" unless (-d "$directory/del_$record_type");
+ $num_records_deleted = generate_deleted_marc_records($record_type, $sth, "$directory/del_$record_type", $as_xml);
+ mark_zebraqueue_done($record_type, 'deleted');
+ $sth = select_zebraqueue_records($record_type, 'updated');
+ mkdir "$directory/upd_$record_type" unless (-d "$directory/upd_$record_type");
+ $num_records_exported = export_marc_records($record_type, $sth, "$directory/upd_$record_type", $as_xml, $noxml);
+ mark_zebraqueue_done($record_type, 'updated');
+ } else {
+ my $sth = select_all_records($record_type);
+ $num_records_exported = export_marc_records($record_type, $sth, "$directory/$record_type", $as_xml, $noxml);
+ }
+ }
+
+ #
+ # and reindexing everything
+ #
+ print "====================\n";
+ print "REINDEXING zebra\n";
+ print "====================\n";
+ my $record_fmt = ($as_xml) ? 'marcxml' : 'iso2709' ;
+ if ($process_zebraqueue) {
+ do_indexing($record_type, 'delete', "$directory/del_$record_type", $reset, $noshadow, $record_fmt)
+ if $num_records_deleted;
+ do_indexing($record_type, 'update', "$directory/upd_$record_type", $reset, $noshadow, $record_fmt)
+ if $num_records_exported;
+ } else {
+ do_indexing($record_type, 'update', "$directory/$record_type", $reset, $noshadow, $record_fmt)
+ if $num_records_exported;
+ }
+}
+
+sub select_zebraqueue_records {
+ my ($record_type, $update_type) = @_;
+
+ my $server = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
+ my $op = ($update_type eq 'deleted') ? 'recordDelete' : 'specialUpdate';
+
+ my $sth = $dbh->prepare("SELECT DISTINCT biblio_auth_number
+ FROM zebraqueue
+ WHERE server = ?
+ AND operation = ?
+ AND done = 0");
+ $sth->execute($server, $op);
+ return $sth;
+}
+
+sub mark_zebraqueue_done {
+ my ($record_type, $update_type) = @_;
+
+ my $server = ($record_type eq 'biblio') ? 'biblioserver' : 'authorityserver';
+ my $op = ($update_type eq 'deleted') ? 'recordDelete' : 'specialUpdate';
+
+ if ($op eq 'recordDelete') {
+ my $sth = $dbh->prepare("UPDATE zebraqueue SET done = 1
+ WHERE id IN (
+ SELECT id FROM (
+ SELECT z1.id
+ FROM zebraqueue z1
+ JOIN zebraqueue z2 ON z2.biblio_auth_number = z1.biblio_auth_number
+ WHERE z1.done = 0
+ AND z1.server = ?
+ AND z2.done = 0
+ AND z2.server = ?
+ AND z1.operation = ?
+ ) d2
+ )
+ ");
+ $sth->execute($server, $server, $op); # if we've deleted a record, any prior specialUpdates are void
+ } else {
+ my $sth = $dbh->prepare("UPDATE zebraqueue SET done = 1
+ WHERE server = ?
+ AND operation = ?
+ AND done = 0");
+ $sth->execute($server, $op);
+ }
+}
+
+sub select_all_records {
+ my $record_type = shift;
+ return ($record_type eq 'biblio') ? select_all_biblios() : select_all_authorities();
+}
+
+sub select_all_authorities {
+ my $sth = $dbh->prepare("SELECT authid FROM auth_header");
+ $sth->execute();
+ return $sth;
+}
+
+sub select_all_biblios {
+ my $sth = $dbh->prepare("SELECT biblionumber FROM biblioitems ORDER BY biblionumber");
+ $sth->execute();
+ return $sth;
+}
+
+sub export_marc_records {
+ my ($record_type, $sth, $directory, $as_xml, $noxml) = @_;
+
+ my $num_exported = 0;
+ open (OUT, ">:utf8 ", "$directory/exported_records") or die $!;
+ my $i = 0;
+ while (my ($record_number) = $sth->fetchrow_array) {
+ print ".";
+ print "\r$i" unless ($i++ %100);
+ my ($marc) = get_corrected_marc_record($record_type, $record_number, $noxml);
+ if (defined $marc) {
+ # FIXME - when more than one record is exported and $as_xml is true,
+ # the output file is not valid XML - it's just multiple <record> elements
+ # strung together with no single root element. zebraidx doesn't seem
+ # to care, though, at least if you're using the GRS-1 filter. It does
+ # care if you're using the DOM filter, which requires valid XML file(s).
+ print OUT ($as_xml) ? $marc->as_xml_record() : $marc->as_usmarc();
+ $num_exported++;
+ }
+ }
+ print "\nRecords exported: $num_exported\n";
+ close OUT;
+ return $num_exported;
+}
+
+sub generate_deleted_marc_records {
+ my ($record_type, $sth, $directory, $as_xml) = @_;
+
+ my $num_exported = 0;
+ open (OUT, ">:utf8 ", "$directory/exported_records") or die $!;
+ my $i = 0;
+ while (my ($record_number) = $sth->fetchrow_array) {
+ print "\r$i" unless ($i++ %100);
+ print ".";
+
+ my $marc = MARC::Record->new();
+ if ($record_type eq 'biblio') {
+ fix_biblio_ids($marc, $record_number, $record_number);
+ } else {
+ fix_authority_id($marc, $record_number);
+ }
+ if (C4::Context->preference("marcflavour") eq "UNIMARC") {
+ fix_unimarc_100($marc);
+ }
+
+ print OUT ($as_xml) ? $marc->as_xml_record() : $marc->as_usmarc();
+ $num_exported++;
+ }
+ print "\nRecords exported: $num_exported\n";
+ close OUT;
+ return $num_exported;
+
+
+}
+
+sub get_corrected_marc_record {
+ my ($record_type, $record_number, $noxml) = @_;
+
+ my $marc = get_raw_marc_record($record_type, $record_number, $noxml);
+
+ if (defined $marc) {
+ fix_leader($marc);
+ if ($record_type eq 'biblio') {
+ my $succeeded = fix_biblio_ids($marc, $record_number);
+ return unless $succeeded;
+ } else {
+ fix_authority_id($marc, $record_number);
+ }
+ if (C4::Context->preference("marcflavour") eq "UNIMARC") {
+ fix_unimarc_100($marc);
+ }
+ }
+
+ return $marc;
+}
+
+sub get_raw_marc_record {
+ my ($record_type, $record_number, $noxml) = @_;
+
+ my $marc;
+ if ($record_type eq 'biblio') {
+ if ($noxml) {
+ my $fetch_sth = $dbh->prepare_cached("SELECT marc FROM biblioitems WHERE biblionumber = ?");
+ $fetch_sth->execute($record_number);
+ if (my ($blob) = $fetch_sth->fetchrow_array) {
+ $marc = MARC::Record->new_from_usmarc($blob);
+ } else {
+ warn "failed to retrieve biblio $record_number";
+ }
+ $fetch_sth->finish();
+ } else {
+ eval { $marc = GetMarcBiblio($record_number); };
+ if ($@) {
+ warn "failed to retrieve biblio $record_number";
+ return;
+ }
+ }
+ } else {
+ eval { $marc = GetAuthority($record_number); };
+ if ($@) {
+ warn "failed to retrieve authority $record_number";
+ return;
+ }
+ }
+ return $marc;
+}
+
+sub fix_leader {
+ # FIXME - this routine is suspect
+ # It blanks the Leader/00-05 and Leader/12-16 to
+ # force them to be recalculated correct when
+ # the $marc->as_usmarc() or $marc->as_xml() is called.
+ # But why is this necessary? It would be a serious bug
+ # in MARC::Record (definitely) and MARC::File::XML (arguably)
+ # if they are emitting incorrect leader values.
+ my $marc = shift;
+
+ my $leader = $marc->leader;
+ substr($leader, 0, 5) = ' ';
+ substr($leader, 10, 7) = '22 ';
+ $marc->leader(substr($leader, 0, 24));
+}
+
+sub fix_biblio_ids {
+ # FIXME - it is essential to ensure that the biblionumber is present,
+ # otherwise, Zebra will choke on the record. However, this
+ # logic belongs in the relevant C4::Biblio APIs.
+ my ($marc, $biblionumber) = @_;
+ my $biblioitemnumber;
+ if (@_) {
+ $biblioitemnumber = shift;
+ } else {
+ my $sth = $dbh->prepare(
+ "SELECT biblioitemnumber FROM biblioitems WHERE biblionumber=?");
+ $sth->execute($biblionumber);
+ ($biblioitemnumber) = $sth->fetchrow_array;
+ $sth->finish;
+ unless ($biblioitemnumber) {
+ warn "failed to get biblioitemnumber for biblio $biblionumber";
+ return 0;
+ }
+ }
+
+ # FIXME - this is cheating on two levels
+ # 1. C4::Biblio::_koha_marc_update_bib_ids is meant to be an internal function
+ # 2. Making sure that the biblionumber and biblioitemnumber are correct and
+ # present in the MARC::Record object ought to be part of GetMarcBiblio.
+ #
+ # On the other hand, this better for now than what rebuild_zebra.pl used to
+ # do, which was duplicate the code for inserting the biblionumber
+ # and biblioitemnumber
+ C4::Biblio::_koha_marc_update_bib_ids($marc, '', $biblionumber, $biblioitemnumber);
+
+ return 1;
+}
+
+sub fix_authority_id {
+ # FIXME - as with fix_biblio_ids, the authid must be present
+ # for Zebra's sake. However, this really belongs
+ # in C4::AuthoritiesMarc.
+ my ($marc, $authid) = @_;
+ unless ($marc->field('001') and $marc->field('001')->data() eq $authid){
+ $marc->delete_field($marc->field('001'));
+ $marc->insert_fields_ordered(MARC::Field->new('001',$authid));
+ }
+}
+
+sub fix_unimarc_100 {
+ # FIXME - again, if this is necessary, it belongs in C4::AuthoritiesMarc.
+ my $marc = shift;
+
+ my $string;
+ if ( length($marc->subfield( 100, "a" )) == 35 ) {
+ $string = $marc->subfield( 100, "a" );
+ my $f100 = $marc->field(100);
+ $marc->delete_field($f100);
+ }
+ else {
+ $string = POSIX::strftime( "%Y%m%d", localtime );
+ $string =~ s/\-//g;
+ $string = sprintf( "%-*s", 35, $string );
+ }
+ substr( $string, 22, 6, "frey50" );
+ unless ( length($marc->subfield( 100, "a" )) == 35 ) {
+ $marc->delete_field($marc->field(100));
+ $marc->insert_grouped_field(MARC::Field->new( 100, "", "", "a" => $string ));
+ }
+}
+
sub do_indexing {
my ($record_type, $op, $record_dir, $reset_index, $noshadow, $record_format) = @_;
-a index authority records
+ -z select only updated and deleted
+ records marked in the zebraqueue
+ table. Cannot be used with -r
+ or -s.
+
-r clear Zebra index before
adding records to index