X-Git-Url: http://koha-dev.rot13.org:8081/gitweb/?a=blobdiff_plain;f=C4%2FImportBatch.pm;h=5f466134e2976090a0f3f5c4092a51fc44671aab;hb=6e1ec2472a140103d55bdaf4cf9fe661485b5af7;hp=d0bfc72a38773e3ef5664dbc6cad2da426b35ddc;hpb=ae5b76c89b8a61ad82528ea85d1c3673f3318a09;p=koha_fer diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index d0bfc72a38..5f466134e2 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -1,6 +1,6 @@ package C4::ImportBatch; -# Copyright (C) 2007 LibLime +# Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services # # This file is part of Koha. # @@ -13,9 +13,9 @@ package C4::ImportBatch; # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR # A PARTICULAR PURPOSE. See the GNU General Public License for more details. # -# You should have received a copy of the GNU General Public License along with -# Koha; if not, write to the Free Software Foundation, Inc., 59 Temple Place, -# Suite 330, Boston, MA 02111-1307 USA +# You should have received a copy of the GNU General Public License along +# with Koha; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. use strict; use warnings; @@ -25,32 +25,39 @@ use C4::Koha; use C4::Biblio; use C4::Items; use C4::Charset; +use C4::AuthoritiesMarc; use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); BEGIN { # set the version for version checking - $VERSION = 3.01; + $VERSION = 3.07.00.049; require Exporter; @ISA = qw(Exporter); @EXPORT = qw( GetZ3950BatchId + GetWebserviceBatchId GetImportRecordMarc + GetImportRecordMarcXML AddImportBatch GetImportBatch + AddAuthToBatch AddBiblioToBatch + AddItemsToImportBiblio + ModAuthorityInBatch ModBiblioInBatch BatchStageMarcRecords - BatchFindBibDuplicates - BatchCommitBibRecords - BatchRevertBibRecords + BatchFindDuplicates + BatchCommitRecords + BatchRevertRecords CleanBatch GetAllImportBatches + GetStagedWebserviceBatches GetImportBatchRangeDesc GetNumberOfNonZ3950ImportBatches - GetImportBibliosRange + GetImportRecordsRange GetItemNumbersFromImportBatch GetImportBatchStatus @@ -78,21 +85,13 @@ C4::ImportBatch - manage batches of imported MARC records =head1 SYNOPSIS -=over 4 - use C4::ImportBatch; -=back - =head1 FUNCTIONS =head2 GetZ3950BatchId -=over 4 - -my $batchid = GetZ3950BatchId($z3950server); - -=back + my $batchid = GetZ3950BatchId($z3950server); Retrieves the ID of the import batch for the Z39.50 reservoir for the given target. If necessary, @@ -113,19 +112,54 @@ sub GetZ3950BatchId { if (defined $rowref) { return $rowref->[0]; } else { - my $batch_id = AddImportBatch('create_new', 'staged', 'z3950', $z3950server, ''); + my $batch_id = AddImportBatch( { + overlay_action => 'create_new', + import_status => 'staged', + batch_type => 'z3950', + file_name => $z3950server, + } ); return $batch_id; } } -=head2 GetImportRecordMarc +=head2 GetWebserviceBatchId + + my $batchid = GetWebserviceBatchId(); + +Retrieves the ID of the import batch for webservice. +If necessary, creates the import batch. + +=cut + +my $WEBSERVICE_BASE_QRY = <dbh; + my $sql = $WEBSERVICE_BASE_QRY; + my @args; + foreach my $field (qw(matcher_id overlay_action nomatch_action item_action)) { + if (my $val = $params->{$field}) { + $sql .= " AND $field = ?"; + push @args, $val; + } + } + my $id = $dbh->selectrow_array($sql, undef, @args); + return $id if $id; -=over 4 + $params->{batch_type} = 'webservice'; + $params->{import_status} = 'staged'; + return AddImportBatch($params); +} -my ($marcblob, $encoding) = GetImportRecordMarc($import_record_id); +=head2 GetImportRecordMarc -=back + my ($marcblob, $encoding) = GetImportRecordMarc($import_record_id); =cut @@ -137,42 +171,57 @@ sub GetImportRecordMarc { $sth->execute($import_record_id); my ($marc, $encoding) = $sth->fetchrow(); $sth->finish(); - return $marc; + return $marc, $encoding; } -=head2 AddImportBatch +=head2 GetImportRecordMarcXML -=over 4 - -my $batch_id = AddImportBatch($overlay_action, $import_status, $type, $file_name, $comments); - -=back + my $marcxml = GetImportRecordMarcXML($import_record_id); =cut -sub AddImportBatch { - my ($overlay_action, $import_status, $type, $file_name, $comments) = @_; +sub GetImportRecordMarcXML { + my ($import_record_id) = @_; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("INSERT INTO import_batches (overlay_action, import_status, batch_type, - file_name, comments) - VALUES (?, ?, ?, ?, ?)"); - $sth->execute($overlay_action, $import_status, $type, $file_name, $comments); - my $batch_id = $dbh->{'mysql_insertid'}; + my $sth = $dbh->prepare("SELECT marcxml FROM import_records WHERE import_record_id = ?"); + $sth->execute($import_record_id); + my ($marcxml) = $sth->fetchrow(); $sth->finish(); - - return $batch_id; + return $marcxml; } -=head2 GetImportBatch +=head2 AddImportBatch + + my $batch_id = AddImportBatch($params_hash); -=over 4 +=cut -my $row = GetImportBatch($batch_id); +sub AddImportBatch { + my ($params) = @_; + + my (@fields, @vals); + foreach (qw( matcher_id template_id branchcode + overlay_action nomatch_action item_action + import_status batch_type file_name comments record_type )) { + if (exists $params->{$_}) { + push @fields, $_; + push @vals, $params->{$_}; + } + } + my $dbh = C4::Context->dbh; + $dbh->do("INSERT INTO import_batches (".join( ',', @fields).") + VALUES (".join( ',', map '?', @fields).")", + undef, + @vals); + return $dbh->{'mysql_insertid'}; +} -=back +=head2 GetImportBatch + + my $row = GetImportBatch($batch_id); Retrieve a hashref of an import_batches row. @@ -193,11 +242,8 @@ sub GetImportBatch { =head2 AddBiblioToBatch -=over 4 - -my $import_record_id = AddBiblioToBatch($batch_id, $record_sequence, $marc_record, $encoding, $z3950random, $update_counts); - -=back + my $import_record_id = AddBiblioToBatch($batch_id, $record_sequence, + $marc_record, $encoding, $z3950random, $update_counts); =cut @@ -209,7 +255,7 @@ sub AddBiblioToBatch { my $z3950random = shift; my $update_counts = @_ ? shift : 1; - my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, $z3950random); + my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, $z3950random, C4::Context->preference('marcflavour')); _add_biblio_fields($import_record_id, $marc_record); _update_batch_record_counts($batch_id) if $update_counts; return $import_record_id; @@ -217,45 +263,76 @@ sub AddBiblioToBatch { =head2 ModBiblioInBatch -=over 4 - -ModBiblioInBatch($import_record_id, $marc_record); - -=back + ModBiblioInBatch($import_record_id, $marc_record); =cut sub ModBiblioInBatch { my ($import_record_id, $marc_record) = @_; - _update_import_record_marc($import_record_id, $marc_record); + _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour')); _update_biblio_fields($import_record_id, $marc_record); } -=head2 BatchStageMarcRecords +=head2 AddAuthToBatch -=over 4 + my $import_record_id = AddAuthToBatch($batch_id, $record_sequence, + $marc_record, $encoding, $z3950random, $update_counts, [$marc_type]); -($batch_id, $num_records, $num_items, @invalid_records) = - BatchStageMarcRecords($marc_flavor, $marc_records, $file_name, +=cut + +sub AddAuthToBatch { + my $batch_id = shift; + my $record_sequence = shift; + my $marc_record = shift; + my $encoding = shift; + my $z3950random = shift; + my $update_counts = @_ ? shift : 1; + my $marc_type = shift || C4::Context->preference('marcflavour'); + + $marc_type = 'UNIMARCAUTH' if $marc_type eq 'UNIMARC'; + + my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random, $marc_type); + _add_auth_fields($import_record_id, $marc_record); + _update_batch_record_counts($batch_id) if $update_counts; + return $import_record_id; +} + +=head2 ModAuthInBatch + + ModAuthInBatch($import_record_id, $marc_record); + +=cut + +sub ModAuthInBatch { + my ($import_record_id, $marc_record) = @_; + + my $marcflavour = C4::Context->preference('marcflavour'); + _update_import_record_marc($import_record_id, $marc_record, $marcflavour eq 'UNIMARC' ? 'UNIMARCAUTH' : 'USMARC'); + +} + +=head2 BatchStageMarcRecords + + ($batch_id, $num_records, $num_items, @invalid_records) = + BatchStageMarcRecords($record_type, $encoding, $marc_records, $file_name, $comments, $branch_code, $parse_items, $leave_as_staging, $progress_interval, $progress_callback); -=back - =cut sub BatchStageMarcRecords { - my $marc_flavor = shift; + my $record_type = shift; + my $encoding = shift; my $marc_records = shift; my $file_name = shift; my $comments = shift; my $branch_code = shift; my $parse_items = shift; my $leave_as_staging = shift; - + # optional callback to monitor status # of job my $progress_interval = 0; @@ -267,13 +344,22 @@ sub BatchStageMarcRecords { $progress_interval = 0 unless 'CODE' eq ref $progress_callback; } - my $batch_id = AddImportBatch('create_new', 'staging', 'batch', $file_name, $comments); + my $batch_id = AddImportBatch( { + overlay_action => 'create_new', + import_status => 'staging', + batch_type => 'batch', + file_name => $file_name, + comments => $comments, + record_type => $record_type, + } ); if ($parse_items) { SetImportBatchItemAction($batch_id, 'always_add'); } else { SetImportBatchItemAction($batch_id, 'ignore'); } + my $marc_type = C4::Context->preference('marcflavour'); + $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth'); my @invalid_records = (); my $num_valid = 0; my $num_items = 0; @@ -288,16 +374,27 @@ sub BatchStageMarcRecords { &$progress_callback($rec_num); } my ($marc_record, $charset_guessed, $char_errors) = - MarcToUTF8Record($marc_blob, C4::Context->preference("marcflavour")); + MarcToUTF8Record($marc_blob, $marc_type, $encoding); + + $encoding = $charset_guessed unless $encoding; + my $import_record_id; if (scalar($marc_record->fields()) == 0) { push @invalid_records, $marc_blob; } else { + + # Normalize the record so it doesn't have separated diacritics + SetUTF8Flag($marc_record); + $num_valid++; - $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $marc_flavor, int(rand(99999)), 0); - if ($parse_items) { - my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0); - $num_items += scalar(@import_items_ids); + if ($record_type eq 'biblio') { + $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); + if ($parse_items) { + my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0); + $num_items += scalar(@import_items_ids); + } + } elsif ($record_type eq 'auth') { + $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0, $marc_type); } } } @@ -311,11 +408,8 @@ sub BatchStageMarcRecords { =head2 AddItemsToImportBiblio -=over 4 - -my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, $update_counts); - -=back + my @import_items_ids = AddItemsToImportBiblio($batch_id, + $import_record_id, $marc_record, $update_counts); =cut @@ -346,18 +440,15 @@ sub AddItemsToImportBiblio { if ($#import_items_ids > -1) { _update_batch_record_counts($batch_id) if $update_counts; - _update_import_record_marc($import_record_id, $marc_record); + _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour')); } return @import_items_ids; } -=head2 BatchFindBibDuplicates - -=over 4 +=head2 BatchFindDuplicates -my $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, $max_matches, $progress_interval, $progress_callback); - -=back + my $num_with_matches = BatchFindDuplicates($batch_id, $matcher, + $max_matches, $progress_interval, $progress_callback); Goes through the records loaded in the batch and attempts to find duplicates for each one. Sets the matching status @@ -374,7 +465,7 @@ singular argument. =cut -sub BatchFindBibDuplicates { +sub BatchFindDuplicates { my $batch_id = shift; my $matcher = shift; my $max_matches = @_ ? shift : 10; @@ -392,9 +483,8 @@ sub BatchFindBibDuplicates { my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT import_record_id, marc + my $sth = $dbh->prepare("SELECT import_record_id, record_type, marc FROM import_records - JOIN import_biblios USING (import_record_id) WHERE import_batch_id = ?"); $sth->execute($batch_id); my $num_with_matches = 0; @@ -422,19 +512,17 @@ sub BatchFindBibDuplicates { return $num_with_matches; } -=head2 BatchCommitBibRecords - -=over 4 - -my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = - BatchCommitBibRecords($batch_id, $progress_interval, $progress_callback); +=head2 BatchCommitRecords -=back + my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = + BatchCommitRecords($batch_id, $framework, + $progress_interval, $progress_callback); =cut -sub BatchCommitBibRecords { +sub BatchCommitRecords { my $batch_id = shift; + my $framework = shift; # optional callback to monitor status # of job @@ -447,25 +535,30 @@ sub BatchCommitBibRecords { $progress_interval = 0 unless 'CODE' eq ref $progress_callback; } + my $record_type; my $num_added = 0; my $num_updated = 0; my $num_items_added = 0; my $num_items_errored = 0; my $num_ignored = 0; # commit (i.e., save, all records in the batch) - # FIXME biblio only at the moment SetImportBatchStatus('importing'); my $overlay_action = GetImportBatchOverlayAction($batch_id); my $nomatch_action = GetImportBatchNoMatchAction($batch_id); my $item_action = GetImportBatchItemAction($batch_id); + my $item_tag; + my $item_subfield; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marc, encoding + my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marc, encoding FROM import_records - JOIN import_biblios USING (import_record_id) + LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id) + LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id) WHERE import_batch_id = ?"); $sth->execute($batch_id); + my $marcflavour = C4::Context->preference('marcflavour'); my $rec_num = 0; while (my $rowref = $sth->fetchrow_hashref) { + $record_type = $rowref->{'record_type'}; $rec_num++; if ($progress_interval and (0 == ($rec_num % $progress_interval))) { &$progress_callback($rec_num); @@ -475,69 +568,98 @@ sub BatchCommitBibRecords { next; } + my $marc_type; + if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') { + $marc_type = 'UNIMARCAUTH'; + } elsif ($marcflavour eq 'UNIMARC') { + $marc_type = 'UNIMARC'; + } else { + $marc_type = 'USMARC'; + } my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'}); - # remove any item tags - rely on BatchCommitItems - my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",''); - foreach my $item_field ($marc_record->field($item_tag)) { - $marc_record->delete_field($item_field); + if ($record_type eq 'biblio') { + # remove any item tags - rely on BatchCommitItems + ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",''); + foreach my $item_field ($marc_record->field($item_tag)) { + $marc_record->delete_field($item_field); + } } - # decide what what to do with the bib and item records - my ($bib_result, $item_result, $bib_match) = + my ($record_result, $item_result, $record_match) = _get_commit_action($overlay_action, $nomatch_action, $item_action, - $rowref->{'overlay_status'}, $rowref->{'import_record_id'}); + $rowref->{'overlay_status'}, $rowref->{'import_record_id'}, $record_type); - if ($bib_result eq 'create_new') { + my $recordid; + my $query; + if ($record_result eq 'create_new') { $num_added++; - my ($biblionumber, $biblioitemnumber) = AddBiblio($marc_record, ''); - my $sth = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); - $sth->execute($biblionumber, $rowref->{'import_record_id'}); - $sth->finish(); - if ($item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); - $num_items_added += $bib_items_added; - $num_items_errored += $bib_items_errored; + if ($record_type eq 'biblio') { + my $biblioitemnumber; + ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework); + $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; + if ($item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + } + } else { + $recordid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record)); + $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?"; } + my $sth = $dbh->prepare_cached($query); + $sth->execute($recordid, $rowref->{'import_record_id'}); + $sth->finish(); SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); - } elsif ($bib_result eq 'replace') { + } elsif ($record_result eq 'replace') { $num_updated++; - my $biblionumber = $bib_match; - my ($count, $oldbiblio) = GetBiblio($biblionumber); - my $oldxml = GetXmlBiblio($biblionumber); - - # remove item fields so that they don't get - # added again if record is reverted - my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}); - foreach my $item_field ($old_marc->field($item_tag)) { - $old_marc->delete_field($item_field); - } + $recordid = $record_match; + my $oldxml; + if ($record_type eq 'biblio') { + my ($count, $oldbiblio) = GetBiblio($recordid); + $oldxml = GetXmlBiblio($recordid); + + # remove item fields so that they don't get + # added again if record is reverted + # FIXME: GetXmlBiblio output should not contain item info any more! So the next foreach should not be needed. Does not hurt either; may remove old 952s that should not have been there anymore. + my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}, $marc_type); + foreach my $item_field ($old_marc->field($item_tag)) { + $old_marc->delete_field($item_field); + } + $oldxml = $old_marc->as_xml($marc_type); + + ModBiblio($marc_record, $recordid, $oldbiblio->{'frameworkcode'}); + $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; + + if ($item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); + $num_items_added += $bib_items_added; + $num_items_errored += $bib_items_errored; + } + } else { + $oldxml = GetAuthorityXML($recordid); - ModBiblio($marc_record, $biblionumber, $oldbiblio->{'frameworkcode'}); + ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record)); + $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?"; + } my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ? WHERE import_record_id = ?"); - $sth->execute($old_marc->as_xml(), $rowref->{'import_record_id'}); + $sth->execute($oldxml, $rowref->{'import_record_id'}); $sth->finish(); - my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); - $sth2->execute($biblionumber, $rowref->{'import_record_id'}); + my $sth2 = $dbh->prepare_cached($query); + $sth2->execute($recordid, $rowref->{'import_record_id'}); $sth2->finish(); - if ($item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); - $num_items_added += $bib_items_added; - $num_items_errored += $bib_items_errored; - } SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); - } elsif ($bib_result eq 'ignore') { + } elsif ($record_result eq 'ignore') { $num_ignored++; - my $biblionumber = $bib_match; - if (defined $biblionumber and $item_result eq 'create_new') { - my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $biblionumber); + if ($record_type eq 'biblio' and defined $recordid and $item_result eq 'create_new') { + my ($bib_items_added, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid); $num_items_added += $bib_items_added; $num_items_errored += $bib_items_errored; # still need to record the matched biblionumber so that the # items can be reverted my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); - $sth2->execute($biblionumber, $rowref->{'import_record_id'}); + $sth2->execute($recordid, $rowref->{'import_record_id'}); SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); } SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored'); @@ -550,11 +672,8 @@ sub BatchCommitBibRecords { =head2 BatchCommitItems -=over 4 - -($num_items_added, $num_items_errored) = BatchCommitItems($import_record_id, $biblionumber); - -=back + ($num_items_added, $num_items_errored) = + BatchCommitItems($import_record_id, $biblionumber); =cut @@ -599,65 +718,93 @@ sub BatchCommitItems { return ($num_items_added, $num_items_errored); } -=head2 BatchRevertBibRecords +=head2 BatchRevertRecords -=over 4 - -my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) = BatchRevertBibRecords($batch_id); - -=back + my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, + $num_ignored) = BatchRevertRecords($batch_id); =cut -sub BatchRevertBibRecords { +sub BatchRevertRecords { my $batch_id = shift; + my $record_type; my $num_deleted = 0; my $num_errors = 0; my $num_reverted = 0; - my $num_items_deleted = 0; my $num_ignored = 0; + my $num_items_deleted = 0; # commit (i.e., save, all records in the batch) - # FIXME biblio only at the moment SetImportBatchStatus('reverting'); my $overlay_action = GetImportBatchOverlayAction($batch_id); my $nomatch_action = GetImportBatchNoMatchAction($batch_id); my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT import_record_id, status, overlay_status, marcxml_old, encoding, matched_biblionumber + my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marcxml_old, encoding, matched_biblionumber, matched_authid FROM import_records - JOIN import_biblios USING (import_record_id) + LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id) + LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id) WHERE import_batch_id = ?"); $sth->execute($batch_id); + my $marc_type; + my $marcflavour = C4::Context->preference('marcflavour'); while (my $rowref = $sth->fetchrow_hashref) { + $record_type = $rowref->{'record_type'}; if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') { $num_ignored++; next; } + if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') { + $marc_type = 'UNIMARCAUTH'; + } elsif ($marcflavour eq 'UNIMARC') { + $marc_type = 'UNIMARC'; + } else { + $marc_type = 'USMARC'; + } - my $bib_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'}); + my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'}); - if ($bib_result eq 'delete') { - $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); - my $error = DelBiblio($rowref->{'matched_biblionumber'}); + if ($record_result eq 'delete') { + my $error = undef; + if ($record_type eq 'biblio') { + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + $error = DelBiblio($rowref->{'matched_biblionumber'}); + } else { + my $deletedauthid = DelAuthority($rowref->{'matched_authid'}); + } if (defined $error) { $num_errors++; } else { $num_deleted++; SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); } - } elsif ($bib_result eq 'restore') { + } elsif ($record_result eq 'restore') { $num_reverted++; - my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}); - my $biblionumber = $rowref->{'matched_biblionumber'}; - my ($count, $oldbiblio) = GetBiblio($biblionumber); - $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); - ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'}); + my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}, $marc_type); + if ($record_type eq 'biblio') { + my $biblionumber = $rowref->{'matched_biblionumber'}; + my ($count, $oldbiblio) = GetBiblio($biblionumber); + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'}); + } else { + my $authid = $rowref->{'matched_authid'}; + ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record)); + } SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); - } elsif ($bib_result eq 'ignore') { - $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + } elsif ($record_result eq 'ignore') { + if ($record_type eq 'biblio') { + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); + } SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted'); } - my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?"); + my $query; + if ($record_type eq 'biblio') { + # remove matched_biblionumber only if there is no 'imported' item left + $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?"; + $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ? AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')"; + } else { + $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?"; + } + my $sth2 = $dbh->prepare_cached($query); $sth2->execute($rowref->{'import_record_id'}); } @@ -668,11 +815,7 @@ sub BatchRevertBibRecords { =head2 BatchRevertItems -=over 4 - -my $num_items_deleted = BatchRevertItems($import_record_id, $biblionumber); - -=back + my $num_items_deleted = BatchRevertItems($import_record_id, $biblionumber); =cut @@ -689,13 +832,18 @@ sub BatchRevertItems { $sth->bind_param(1, $import_record_id); $sth->execute(); while (my $row = $sth->fetchrow_hashref()) { - DelItem($dbh, $biblionumber, $row->{'itemnumber'}); - my $updsth = $dbh->prepare("UPDATE import_items SET status = ? WHERE import_items_id = ?"); - $updsth->bind_param(1, 'reverted'); - $updsth->bind_param(2, $row->{'import_items_id'}); - $updsth->execute(); - $updsth->finish(); - $num_items_deleted++; + my $error = DelItemCheck($dbh, $biblionumber, $row->{'itemnumber'}); + if ($error == 1){ + my $updsth = $dbh->prepare("UPDATE import_items SET status = ? WHERE import_items_id = ?"); + $updsth->bind_param(1, 'reverted'); + $updsth->bind_param(2, $row->{'import_items_id'}); + $updsth->execute(); + $updsth->finish(); + $num_items_deleted++; + } + else { + next; + } } $sth->finish(); return $num_items_deleted; @@ -703,11 +851,7 @@ sub BatchRevertItems { =head2 CleanBatch -=over 4 - -CleanBatch($batch_id) - -=back + CleanBatch($batch_id) Deletes all staged records from the import batch and sets the status of the batch to 'cleaned'. Note @@ -726,11 +870,7 @@ sub CleanBatch { =head2 GetAllImportBatches -=over 4 - -my $results = GetAllImportBatches(); - -=back + my $results = GetAllImportBatches(); Returns a references to an array of hash references corresponding to all import_batches rows (of batch_type 'batch'), sorted in @@ -741,7 +881,7 @@ ascending order by import_batch_id. sub GetAllImportBatches { my $dbh = C4::Context->dbh; my $sth = $dbh->prepare_cached("SELECT * FROM import_batches - WHERE batch_type = 'batch' + WHERE batch_type IN ('batch', 'webservice') ORDER BY import_batch_id ASC"); my $results = []; @@ -753,13 +893,28 @@ sub GetAllImportBatches { return $results; } -=head2 GetImportBatchRangeDesc +=head2 GetStagedWebserviceBatches -=over 4 + my $batch_ids = GetStagedWebserviceBatches(); -my $results = GetImportBatchRangeDesc($offset, $results_per_group); +Returns a references to an array of batch id's +of batch_type 'webservice' that are not imported -=back +=cut + +my $PENDING_WEBSERVICE_BATCHES_QRY = <dbh; + return $dbh->selectcol_arrayref($PENDING_WEBSERVICE_BATCHES_QRY); +} + +=head2 GetImportBatchRangeDesc + + my $results = GetImportBatchRangeDesc($offset, $results_per_group); Returns a reference to an array of hash references corresponding to import_batches rows (sorted in descending order by import_batch_id) @@ -771,30 +926,35 @@ sub GetImportBatchRangeDesc { my ($offset, $results_per_group) = @_; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare_cached("SELECT * FROM import_batches - WHERE batch_type = 'batch' - ORDER BY import_batch_id DESC - LIMIT ? OFFSET ?"); - $sth->bind_param(1, $results_per_group); - $sth->bind_param(2, $offset); - - my $results = []; - $sth->execute(); - while (my $row = $sth->fetchrow_hashref) { - push @$results, $row; + my $query = "SELECT * FROM import_batches + WHERE batch_type IN ('batch', 'webservice') + ORDER BY import_batch_id DESC"; + my @params; + if ($results_per_group){ + $query .= " LIMIT ?"; + push(@params, $results_per_group); + } + if ($offset){ + $query .= " OFFSET ?"; + push(@params, $offset); } + my $sth = $dbh->prepare_cached($query); + $sth->execute(@params); + my $results = $sth->fetchall_arrayref({}); $sth->finish(); return $results; } =head2 GetItemNumbersFromImportBatch + my @itemsnos = GetItemNumbersFromImportBatch($batch_id); + =cut sub GetItemNumbersFromImportBatch { my ($batch_id) = @_; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("select itemnumber from import_batches,import_records,import_items where import_batches.import_batch_id=import_records.import_batch_id and import_records.import_record_id=import_items.import_record_id and import_batches.import_batch_id=?"); + my $sth = $dbh->prepare("SELECT itemnumber FROM import_batches,import_records,import_items WHERE import_batches.import_batch_id=import_records.import_batch_id AND import_records.import_record_id=import_items.import_record_id AND import_batches.import_batch_id=?"); $sth->execute($batch_id); my @items ; while ( my ($itm) = $sth->fetchrow_array ) { @@ -805,55 +965,59 @@ sub GetItemNumbersFromImportBatch { =head2 GetNumberOfImportBatches -=over 4 - -my $count = GetNumberOfImportBatches(); - -=back + my $count = GetNumberOfImportBatches(); =cut sub GetNumberOfNonZ3950ImportBatches { my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT COUNT(*) FROM import_batches WHERE batch_type='batch'"); + my $sth = $dbh->prepare("SELECT COUNT(*) FROM import_batches WHERE batch_type != 'z3950'"); $sth->execute(); my ($count) = $sth->fetchrow_array(); $sth->finish(); return $count; } -=head2 GetImportBibliosRange +=head2 GetImportRecordsRange -=over 4 - -my $results = GetImportBibliosRange($batch_id, $offset, $results_per_group); - -=back + my $results = GetImportRecordsRange($batch_id, $offset, $results_per_group); Returns a reference to an array of hash references corresponding to -import_biblios/import_records rows for a given batch +import_biblios/import_auths/import_records rows for a given batch starting at the given offset. =cut -sub GetImportBibliosRange { - my ($batch_id, $offset, $results_per_group) = @_; +sub GetImportRecordsRange { + my ($batch_id, $offset, $results_per_group, $status) = @_; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare_cached("SELECT title, author, isbn, issn, import_record_id, record_sequence, - matched_biblionumber, status, overlay_status + my $query = "SELECT title, author, isbn, issn, authorized_heading, import_records.import_record_id, + record_sequence, status, overlay_status, + matched_biblionumber, matched_authid, record_type FROM import_records - JOIN import_biblios USING (import_record_id) - WHERE import_batch_id = ? - ORDER BY import_record_id LIMIT ? OFFSET ?"); - $sth->bind_param(1, $batch_id); - $sth->bind_param(2, $results_per_group); - $sth->bind_param(3, $offset); - my $results = []; - $sth->execute(); - while (my $row = $sth->fetchrow_hashref) { - push @$results, $row; + LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id) + LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id) + WHERE import_batch_id = ?"; + my @params; + push(@params, $batch_id); + if ($status) { + $query .= " AND status=?"; + push(@params,$status); + } + $query.=" ORDER BY import_record_id"; + + if($results_per_group){ + $query .= " LIMIT ?"; + push(@params, $results_per_group); + } + if($offset){ + $query .= " OFFSET ?"; + push(@params, $offset); } + my $sth = $dbh->prepare_cached($query); + $sth->execute(@params); + my $results = $sth->fetchall_arrayref({}); $sth->finish(); return $results; @@ -861,11 +1025,7 @@ sub GetImportBibliosRange { =head2 GetBestRecordMatch -=over 4 - -my $record_id = GetBestRecordMatch($import_record_id); - -=back + my $record_id = GetBestRecordMatch($import_record_id); =cut @@ -885,11 +1045,7 @@ sub GetBestRecordMatch { =head2 GetImportBatchStatus -=over 4 - -my $status = GetImportBatchStatus($batch_id); - -=back + my $status = GetImportBatchStatus($batch_id); =cut @@ -907,11 +1063,7 @@ sub GetImportBatchStatus { =head2 SetImportBatchStatus -=over 4 - -SetImportBatchStatus($batch_id, $new_status); - -=back + SetImportBatchStatus($batch_id, $new_status); =cut @@ -927,11 +1079,7 @@ sub SetImportBatchStatus { =head2 GetImportBatchOverlayAction -=over 4 - -my $overlay_action = GetImportBatchOverlayAction($batch_id); - -=back + my $overlay_action = GetImportBatchOverlayAction($batch_id); =cut @@ -950,11 +1098,7 @@ sub GetImportBatchOverlayAction { =head2 SetImportBatchOverlayAction -=over 4 - -SetImportBatchOverlayAction($batch_id, $new_overlay_action); - -=back + SetImportBatchOverlayAction($batch_id, $new_overlay_action); =cut @@ -970,11 +1114,7 @@ sub SetImportBatchOverlayAction { =head2 GetImportBatchNoMatchAction -=over 4 - -my $nomatch_action = GetImportBatchNoMatchAction($batch_id); - -=back + my $nomatch_action = GetImportBatchNoMatchAction($batch_id); =cut @@ -993,11 +1133,7 @@ sub GetImportBatchNoMatchAction { =head2 SetImportBatchNoMatchAction -=over 4 - -SetImportBatchNoMatchAction($batch_id, $new_nomatch_action); - -=back + SetImportBatchNoMatchAction($batch_id, $new_nomatch_action); =cut @@ -1013,11 +1149,7 @@ sub SetImportBatchNoMatchAction { =head2 GetImportBatchItemAction -=over 4 - -my $item_action = GetImportBatchItemAction($batch_id); - -=back + my $item_action = GetImportBatchItemAction($batch_id); =cut @@ -1036,11 +1168,7 @@ sub GetImportBatchItemAction { =head2 SetImportBatchItemAction -=over 4 - -SetImportBatchItemAction($batch_id, $new_item_action); - -=back + SetImportBatchItemAction($batch_id, $new_item_action); =cut @@ -1056,11 +1184,7 @@ sub SetImportBatchItemAction { =head2 GetImportBatchMatcher -=over 4 - -my $matcher_id = GetImportBatchMatcher($batch_id); - -=back + my $matcher_id = GetImportBatchMatcher($batch_id); =cut @@ -1079,11 +1203,7 @@ sub GetImportBatchMatcher { =head2 SetImportBatchMatcher -=over 4 - -SetImportBatchMatcher($batch_id, $new_matcher_id); - -=back + SetImportBatchMatcher($batch_id, $new_matcher_id); =cut @@ -1099,11 +1219,7 @@ sub SetImportBatchMatcher { =head2 GetImportRecordOverlayStatus -=over 4 - -my $overlay_status = GetImportRecordOverlayStatus($import_record_id); - -=back + my $overlay_status = GetImportRecordOverlayStatus($import_record_id); =cut @@ -1122,11 +1238,7 @@ sub GetImportRecordOverlayStatus { =head2 SetImportRecordOverlayStatus -=over 4 - -SetImportRecordOverlayStatus($import_record_id, $new_overlay_status); - -=back + SetImportRecordOverlayStatus($import_record_id, $new_overlay_status); =cut @@ -1142,11 +1254,7 @@ sub SetImportRecordOverlayStatus { =head2 GetImportRecordStatus -=over 4 - -my $overlay_status = GetImportRecordStatus($import_record_id); - -=back + my $overlay_status = GetImportRecordStatus($import_record_id); =cut @@ -1165,11 +1273,7 @@ sub GetImportRecordStatus { =head2 SetImportRecordStatus -=over 4 - -SetImportRecordStatus($import_record_id, $new_overlay_status); - -=back + SetImportRecordStatus($import_record_id, $new_overlay_status); =cut @@ -1185,11 +1289,7 @@ sub SetImportRecordStatus { =head2 GetImportRecordMatches -=over 4 - -my $results = GetImportRecordMatches($import_record_id, $best_only); - -=back + my $results = GetImportRecordMatches($import_record_id, $best_only); =cut @@ -1199,16 +1299,21 @@ sub GetImportRecordMatches { my $dbh = C4::Context->dbh; # FIXME currently biblio only - my $sth = $dbh->prepare_cached("SELECT title, author, biblionumber, score + my $sth = $dbh->prepare_cached("SELECT title, author, biblionumber, + candidate_match_id, score, record_type FROM import_records JOIN import_record_matches USING (import_record_id) - JOIN biblio ON (biblionumber = candidate_match_id) + LEFT JOIN biblio ON (biblionumber = candidate_match_id) WHERE import_record_id = ? ORDER BY score DESC, biblionumber DESC"); $sth->bind_param(1, $import_record_id); my $results = []; $sth->execute(); while (my $row = $sth->fetchrow_hashref) { + if ($row->{'record_type'} eq 'auth') { + $row->{'authorized_heading'} = C4::AuthoritiesMarc::GetAuthorizedHeading( { authid => $row->{'candidate_match_id'} } ); + } + next if ($row->{'record_type'} eq 'biblio' && not $row->{'biblionumber'}); push @$results, $row; last if $best_only; } @@ -1221,11 +1326,7 @@ sub GetImportRecordMatches { =head2 SetImportRecordMatches -=over 4 - -SetImportRecordMatches($import_record_id, @matches); - -=back + SetImportRecordMatches($import_record_id, @matches); =cut @@ -1249,13 +1350,13 @@ sub SetImportRecordMatches { # internal functions sub _create_import_record { - my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $z3950random) = @_; + my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $z3950random, $marc_type) = @_; my $dbh = C4::Context->dbh; my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml, record_type, encoding, z3950random) VALUES (?, ?, ?, ?, ?, ?, ?)"); - $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml(), + $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml($marc_type), $record_type, $encoding, $z3950random); my $import_record_id = $dbh->{'mysql_insertid'}; $sth->finish(); @@ -1263,12 +1364,26 @@ sub _create_import_record { } sub _update_import_record_marc { - my ($import_record_id, $marc_record) = @_; + my ($import_record_id, $marc_record, $marc_type) = @_; my $dbh = C4::Context->dbh; my $sth = $dbh->prepare("UPDATE import_records SET marc = ?, marcxml = ? WHERE import_record_id = ?"); - $sth->execute($marc_record->as_usmarc(), $marc_record->as_xml(), $import_record_id); + $sth->execute($marc_record->as_usmarc(), $marc_record->as_xml($marc_type), $import_record_id); + $sth->finish(); +} + +sub _add_auth_fields { + my ($import_record_id, $marc_record) = @_; + + my $controlnumber; + if ($marc_record->field('001')) { + $controlnumber = $marc_record->field('001')->data(); + } + my $authorized_heading = C4::AuthoritiesMarc::GetAuthorizedHeading({ record => $marc_record }); + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare("INSERT INTO import_auths (import_record_id, control_number, authorized_heading) VALUES (?, ?, ?)"); + $sth->execute($import_record_id, $controlnumber, $authorized_heading); $sth->finish(); } @@ -1314,49 +1429,63 @@ sub _update_batch_record_counts { my ($batch_id) = @_; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare_cached("UPDATE import_batches SET num_biblios = ( - SELECT COUNT(*) - FROM import_records - WHERE import_batch_id = import_batches.import_batch_id - AND record_type = 'biblio') + my $sth = $dbh->prepare_cached("UPDATE import_batches SET + num_records = ( + SELECT COUNT(*) + FROM import_records + WHERE import_batch_id = import_batches.import_batch_id), + num_items = ( + SELECT COUNT(*) + FROM import_records + JOIN import_items USING (import_record_id) + WHERE import_batch_id = import_batches.import_batch_id + AND record_type = 'biblio') WHERE import_batch_id = ?"); $sth->bind_param(1, $batch_id); $sth->execute(); $sth->finish(); - $sth = $dbh->prepare_cached("UPDATE import_batches SET num_items = ( - SELECT COUNT(*) - FROM import_records - JOIN import_items USING (import_record_id) - WHERE import_batch_id = import_batches.import_batch_id - AND record_type = 'biblio') - WHERE import_batch_id = ?"); - $sth->bind_param(1, $batch_id); - $sth->execute(); - $sth->finish(); - } sub _get_commit_action { - my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id) = @_; + my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id, $record_type) = @_; - my ($bib_result, $bib_match, $item_result); - - if ($overlay_status ne 'no_match') { - $bib_match = GetBestRecordMatch($import_record_id); - if ($overlay_action eq 'replace') { - $bib_result = defined($bib_match) ? 'replace' : 'create_new'; - } elsif ($overlay_action eq 'create_new') { - $bib_result = 'create_new'; - } elsif ($overlay_action eq 'ignore') { - $bib_result = 'ignore'; - } - $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore'; - } else { - $bib_result = $nomatch_action; - $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore'; - } + if ($record_type eq 'biblio') { + my ($bib_result, $bib_match, $item_result); + + if ($overlay_status ne 'no_match') { + $bib_match = GetBestRecordMatch($import_record_id); + if ($overlay_action eq 'replace') { + $bib_result = defined($bib_match) ? 'replace' : 'create_new'; + } elsif ($overlay_action eq 'create_new') { + $bib_result = 'create_new'; + } elsif ($overlay_action eq 'ignore') { + $bib_result = 'ignore'; + } + $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_matches') ? 'create_new' : 'ignore'; + } else { + $bib_result = $nomatch_action; + $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore'; + } + return ($bib_result, $item_result, $bib_match); + } else { # must be auths + my ($auth_result, $auth_match); + + if ($overlay_status ne 'no_match') { + $auth_match = GetBestRecordMatch($import_record_id); + if ($overlay_action eq 'replace') { + $auth_result = defined($auth_match) ? 'replace' : 'create_new'; + } elsif ($overlay_action eq 'create_new') { + $auth_result = 'create_new'; + } elsif ($overlay_action eq 'ignore') { + $auth_result = 'ignore'; + } + } else { + $auth_result = $nomatch_action; + } + + return ($auth_result, undef, $auth_match); - return ($bib_result, $item_result, $bib_match); + } } sub _get_revert_action { @@ -1381,7 +1510,7 @@ __END__ =head1 AUTHOR -Koha Development Team +Koha Development Team Galen Charlton