X-Git-Url: http://koha-dev.rot13.org:8081/gitweb/?a=blobdiff_plain;f=C4%2FImportBatch.pm;h=ee76020fba74d880e5723848302da64396d205d3;hb=eded6edacc5e3bf8dd0be21ed05842c3b78aadce;hp=f9d7b410fef830afe473bd76f04f39dff2ea34cc;hpb=b527baba5287019a3dbc1407e472f419babf2d8b;p=koha-ffzg.git diff --git a/C4/ImportBatch.pm b/C4/ImportBatch.pm index f9d7b410fe..ee76020fba 100644 --- a/C4/ImportBatch.pm +++ b/C4/ImportBatch.pm @@ -4,79 +4,94 @@ package C4::ImportBatch; # # This file is part of Koha. # -# Koha is free software; you can redistribute it and/or modify it under the -# terms of the GNU General Public License as published by the Free Software -# Foundation; either version 2 of the License, or (at your option) any later -# version. +# Koha is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. # -# Koha is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# Koha is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. # -# You should have received a copy of the GNU General Public License along -# with Koha; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# You should have received a copy of the GNU General Public License +# along with Koha; if not, see . use strict; use warnings; use C4::Context; -use C4::Koha; -use C4::Biblio; -use C4::Items; -use C4::Charset; -use C4::AuthoritiesMarc; - -use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS); - +use C4::Koha qw( GetNormalizedISBN ); +use C4::Biblio qw( + AddBiblio + DelBiblio + GetMarcFromKohaField + GetXmlBiblio + ModBiblio + TransformMarcToKoha +); +use C4::Items qw( AddItemFromMarc ModItemFromMarc ); +use C4::Charset qw( MarcToUTF8Record SetUTF8Flag StripNonXmlChars ); +use C4::AuthoritiesMarc qw( AddAuthority GuessAuthTypeCode GetAuthorityXML ModAuthority DelAuthority ); +use C4::MarcModificationTemplates qw( ModifyRecordWithTemplate ); +use Koha::Items; +use Koha::SearchEngine; +use Koha::SearchEngine::Indexer; +use Koha::Plugins::Handler; +use Koha::Logger; + +our (@ISA, @EXPORT_OK); BEGIN { - # set the version for version checking - $VERSION = 3.07.00.049; - require Exporter; - @ISA = qw(Exporter); - @EXPORT = qw( - GetZ3950BatchId - GetWebserviceBatchId - GetImportRecordMarc - GetImportRecordMarcXML - AddImportBatch - GetImportBatch - AddAuthToBatch - AddBiblioToBatch - AddItemsToImportBiblio - ModAuthorityInBatch - ModBiblioInBatch - - BatchStageMarcRecords - BatchFindDuplicates - BatchCommitRecords - BatchRevertRecords - CleanBatch - - GetAllImportBatches - GetStagedWebserviceBatches - GetImportBatchRangeDesc - GetNumberOfNonZ3950ImportBatches - GetImportRecordsRange - GetItemNumbersFromImportBatch - - GetImportBatchStatus - SetImportBatchStatus - GetImportBatchOverlayAction - SetImportBatchOverlayAction - GetImportBatchNoMatchAction - SetImportBatchNoMatchAction - GetImportBatchItemAction - SetImportBatchItemAction - GetImportBatchMatcher - SetImportBatchMatcher - GetImportRecordOverlayStatus - SetImportRecordOverlayStatus - GetImportRecordStatus - SetImportRecordStatus - GetImportRecordMatches - SetImportRecordMatches - ); + require Exporter; + @ISA = qw(Exporter); + @EXPORT_OK = qw( + GetZ3950BatchId + GetWebserviceBatchId + GetImportRecordMarc + AddImportBatch + GetImportBatch + AddAuthToBatch + AddBiblioToBatch + AddItemsToImportBiblio + ModAuthorityInBatch + + BatchStageMarcRecords + BatchFindDuplicates + BatchCommitRecords + BatchRevertRecords + CleanBatch + DeleteBatch + + GetAllImportBatches + GetStagedWebserviceBatches + GetImportBatchRangeDesc + GetNumberOfNonZ3950ImportBatches + GetImportBiblios + GetImportRecordsRange + GetItemNumbersFromImportBatch + + GetImportBatchStatus + SetImportBatchStatus + GetImportBatchOverlayAction + SetImportBatchOverlayAction + GetImportBatchNoMatchAction + SetImportBatchNoMatchAction + GetImportBatchItemAction + SetImportBatchItemAction + GetImportBatchMatcher + SetImportBatchMatcher + GetImportRecordOverlayStatus + SetImportRecordOverlayStatus + GetImportRecordStatus + SetImportRecordStatus + SetMatchedBiblionumber + GetImportRecordMatches + SetImportRecordMatches + + RecordsFromMARCXMLFile + RecordsFromISO2709File + RecordsFromMarcPlugin + ); } =head1 NAME @@ -167,30 +182,31 @@ sub GetImportRecordMarc { my ($import_record_id) = @_; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT marc, encoding FROM import_records WHERE import_record_id = ?"); - $sth->execute($import_record_id); - my ($marc, $encoding) = $sth->fetchrow(); - $sth->finish(); - return $marc, $encoding; + my ( $marc, $encoding ) = $dbh->selectrow_array(q| + SELECT marc, encoding + FROM import_records + WHERE import_record_id = ? + |, undef, $import_record_id ); + return $marc, $encoding; } -=head2 GetImportRecordMarcXML - - my $marcxml = GetImportRecordMarcXML($import_record_id); - -=cut - -sub GetImportRecordMarcXML { - my ($import_record_id) = @_; - +sub EmbedItemsInImportBiblio { + my ( $record, $import_record_id ) = @_; + my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField( "items.itemnumber" ); my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT marcxml FROM import_records WHERE import_record_id = ?"); - $sth->execute($import_record_id); - my ($marcxml) = $sth->fetchrow(); - $sth->finish(); - return $marcxml; - + my $import_items = $dbh->selectall_arrayref(q| + SELECT import_items.marcxml + FROM import_items + WHERE import_record_id = ? + |, { Slice => {} }, $import_record_id ); + my @item_fields; + for my $import_item ( @$import_items ) { + my $item_marc = MARC::Record::new_from_xml($import_item->{marcxml}, 'UTF-8'); + push @item_fields, $item_marc->field($itemtag); + } + $record->append_fields(@item_fields); + return $record; } =head2 AddImportBatch @@ -231,7 +247,7 @@ sub GetImportBatch { my ($batch_id) = @_; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare_cached("SELECT * FROM import_batches WHERE import_batch_id = ?"); + my $sth = $dbh->prepare_cached("SELECT b.*, p.name as profile FROM import_batches b LEFT JOIN import_batch_profiles p ON p.id = b.profile_id WHERE import_batch_id = ?"); $sth->bind_param(1, $batch_id); $sth->execute(); my $result = $sth->fetchrow_hashref; @@ -243,7 +259,7 @@ sub GetImportBatch { =head2 AddBiblioToBatch my $import_record_id = AddBiblioToBatch($batch_id, $record_sequence, - $marc_record, $encoding, $z3950random, $update_counts); + $marc_record, $encoding, $update_counts); =cut @@ -252,33 +268,18 @@ sub AddBiblioToBatch { my $record_sequence = shift; my $marc_record = shift; my $encoding = shift; - my $z3950random = shift; my $update_counts = @_ ? shift : 1; - my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, $z3950random, C4::Context->preference('marcflavour')); + my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, C4::Context->preference('marcflavour')); _add_biblio_fields($import_record_id, $marc_record); _update_batch_record_counts($batch_id) if $update_counts; return $import_record_id; } -=head2 ModBiblioInBatch - - ModBiblioInBatch($import_record_id, $marc_record); - -=cut - -sub ModBiblioInBatch { - my ($import_record_id, $marc_record) = @_; - - _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour')); - _update_biblio_fields($import_record_id, $marc_record); - -} - =head2 AddAuthToBatch my $import_record_id = AddAuthToBatch($batch_id, $record_sequence, - $marc_record, $encoding, $z3950random, $update_counts, [$marc_type]); + $marc_record, $encoding, $update_counts, [$marc_type]); =cut @@ -287,47 +288,37 @@ sub AddAuthToBatch { my $record_sequence = shift; my $marc_record = shift; my $encoding = shift; - my $z3950random = shift; my $update_counts = @_ ? shift : 1; my $marc_type = shift || C4::Context->preference('marcflavour'); $marc_type = 'UNIMARCAUTH' if $marc_type eq 'UNIMARC'; - my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random, $marc_type); + my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $marc_type); _add_auth_fields($import_record_id, $marc_record); _update_batch_record_counts($batch_id) if $update_counts; return $import_record_id; } -=head2 ModAuthInBatch - - ModAuthInBatch($import_record_id, $marc_record); - -=cut - -sub ModAuthInBatch { - my ($import_record_id, $marc_record) = @_; - - my $marcflavour = C4::Context->preference('marcflavour'); - _update_import_record_marc($import_record_id, $marc_record, $marcflavour eq 'UNIMARC' ? 'UNIMARCAUTH' : 'USMARC'); - -} - =head2 BatchStageMarcRecords - ($batch_id, $num_records, $num_items, @invalid_records) = - BatchStageMarcRecords($record_type, $encoding, $marc_records, $file_name, - $comments, $branch_code, $parse_items, - $leave_as_staging, - $progress_interval, $progress_callback); +( $batch_id, $num_records, $num_items, @invalid_records ) = + BatchStageMarcRecords( + $record_type, $encoding, + $marc_records, $file_name, + $marc_modification_template, $comments, + $branch_code, $parse_items, + $leave_as_staging, $progress_interval, + $progress_callback + ); =cut -sub BatchStageMarcRecords { +sub BatchStageMarcRecords { my $record_type = shift; my $encoding = shift; my $marc_records = shift; my $file_name = shift; + my $marc_modification_template = shift; my $comments = shift; my $branch_code = shift; my $parse_items = shift; @@ -358,6 +349,7 @@ sub BatchStageMarcRecords { SetImportBatchItemAction($batch_id, 'ignore'); } + my $marc_type = C4::Context->preference('marcflavour'); $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth'); my @invalid_records = (); @@ -365,22 +357,17 @@ sub BatchStageMarcRecords { my $num_items = 0; # FIXME - for now, we're dealing only with bibs my $rec_num = 0; - foreach my $marc_blob (split(/\x1D/, $marc_records)) { - $marc_blob =~ s/^\s+//g; - $marc_blob =~ s/\s+$//g; - next unless $marc_blob; + foreach my $marc_record (@$marc_records) { $rec_num++; if ($progress_interval and (0 == ($rec_num % $progress_interval))) { &$progress_callback($rec_num); } - my ($marc_record, $charset_guessed, $char_errors) = - MarcToUTF8Record($marc_blob, $marc_type, $encoding); - $encoding = $charset_guessed unless $encoding; + ModifyRecordWithTemplate( $marc_modification_template, $marc_record ) if ( $marc_modification_template ); my $import_record_id; if (scalar($marc_record->fields()) == 0) { - push @invalid_records, $marc_blob; + push @invalid_records, $marc_record; } else { # Normalize the record so it doesn't have separated diacritics @@ -388,13 +375,13 @@ sub BatchStageMarcRecords { $num_valid++; if ($record_type eq 'biblio') { - $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0); + $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, 0); if ($parse_items) { my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0); $num_items += scalar(@import_items_ids); } } elsif ($record_type eq 'auth') { - $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0, $marc_type); + $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, 0, $marc_type); } } } @@ -422,7 +409,7 @@ sub AddItemsToImportBiblio { my @import_items_ids = (); my $dbh = C4::Context->dbh; - my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",''); + my ($item_tag,$item_subfield) = &GetMarcFromKohaField( "items.itemnumber" ); foreach my $item_field ($marc_record->field($item_tag)) { my $item_marc = MARC::Record->new(); $item_marc->leader("00000 a "); # must set Leader/09 to 'a' @@ -432,7 +419,7 @@ sub AddItemsToImportBiblio { VALUES (?, ?, ?)"); $sth->bind_param(1, $import_record_id); $sth->bind_param(2, 'staged'); - $sth->bind_param(3, $item_marc->as_xml()); + $sth->bind_param(3, $item_marc->as_xml("USMARC")); $sth->execute(); push @import_items_ids, $dbh->{'mysql_insertid'}; $sth->finish(); @@ -440,7 +427,6 @@ sub AddItemsToImportBiblio { if ($#import_items_ids > -1) { _update_batch_record_counts($batch_id) if $update_counts; - _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour')); } return @import_items_ids; } @@ -524,6 +510,8 @@ sub BatchCommitRecords { my $batch_id = shift; my $framework = shift; + my $schema = Koha::Database->schema; + # optional callback to monitor status # of job my $progress_interval = 0; @@ -543,7 +531,7 @@ sub BatchCommitRecords { my $num_items_errored = 0; my $num_ignored = 0; # commit (i.e., save, all records in the batch) - SetImportBatchStatus('importing'); + SetImportBatchStatus($batch_id, 'importing'); my $overlay_action = GetImportBatchOverlayAction($batch_id); my $nomatch_action = GetImportBatchNoMatchAction($batch_id); my $item_action = GetImportBatchItemAction($batch_id); @@ -557,12 +545,23 @@ sub BatchCommitRecords { WHERE import_batch_id = ?"); $sth->execute($batch_id); my $marcflavour = C4::Context->preference('marcflavour'); + + my $userenv = C4::Context->userenv; + my $logged_in_patron = Koha::Patrons->find( $userenv->{number} ); + my $rec_num = 0; + my @biblio_ids; + $schema->txn_begin; # We commit in a transaction while (my $rowref = $sth->fetchrow_hashref) { $record_type = $rowref->{'record_type'}; + $rec_num++; + if ($progress_interval and (0 == ($rec_num % $progress_interval))) { - &$progress_callback($rec_num); + # report progress and commit + $schema->txn_commit; + &$progress_callback( $rec_num ); + $schema->txn_begin; } if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'imported') { $num_ignored++; @@ -580,8 +579,8 @@ sub BatchCommitRecords { my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'}); if ($record_type eq 'biblio') { - # remove any item tags - rely on BatchCommitItems - ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",''); + # remove any item tags - rely on _batchCommitItems + ($item_tag,$item_subfield) = &GetMarcFromKohaField( "items.itemnumber" ); foreach my $item_field ($marc_record->field($item_tag)) { $marc_record->delete_field($item_field); } @@ -597,10 +596,11 @@ sub BatchCommitRecords { $num_added++; if ($record_type eq 'biblio') { my $biblioitemnumber; - ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework); - $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; + ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework, { skip_record_index => 1 }); + push @biblio_ids, $recordid; + $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; # FIXME call SetMatchedBiblionumber instead if ($item_result eq 'create_new' || $item_result eq 'replace') { - my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result); + my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = _batchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result, $biblioitemnumber); $num_items_added += $bib_items_added; $num_items_replaced += $bib_items_replaced; $num_items_errored += $bib_items_errored; @@ -618,7 +618,7 @@ sub BatchCommitRecords { $recordid = $record_match; my $oldxml; if ($record_type eq 'biblio') { - my ($count, $oldbiblio) = GetBiblio($recordid); + my $oldbiblio = Koha::Biblios->find( $recordid ); $oldxml = GetXmlBiblio($recordid); # remove item fields so that they don't get @@ -630,11 +630,26 @@ sub BatchCommitRecords { } $oldxml = $old_marc->as_xml($marc_type); - ModBiblio($marc_record, $recordid, $oldbiblio->{'frameworkcode'}); - $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; + my $context = { source => 'batchimport' }; + if ($logged_in_patron) { + $context->{categorycode} = $logged_in_patron->categorycode; + $context->{userid} = $logged_in_patron->userid; + } + + ModBiblio( + $marc_record, + $recordid, + $oldbiblio->frameworkcode, + { + overlay_context => $context, + skip_record_index => 1 + } + ); + push @biblio_ids, $recordid; + $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"; # FIXME call SetMatchedBiblionumber instead if ($item_result eq 'create_new' || $item_result eq 'replace') { - my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result); + my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = _batchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result); $num_items_added += $bib_items_added; $num_items_replaced += $bib_items_replaced; $num_items_errored += $bib_items_errored; @@ -654,84 +669,118 @@ sub BatchCommitRecords { SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); SetImportRecordStatus($rowref->{'import_record_id'}, 'imported'); } elsif ($record_result eq 'ignore') { - $num_ignored++; $recordid = $record_match; - if ($record_type eq 'biblio' and defined $recordid and $item_result eq 'create_new') { - my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result); + $num_ignored++; + if ($record_type eq 'biblio' and defined $recordid and ( $item_result eq 'create_new' || $item_result eq 'replace' ) ) { + my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = _batchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result); + push @biblio_ids, $recordid if $bib_items_added || $bib_items_replaced; $num_items_added += $bib_items_added; $num_items_replaced += $bib_items_replaced; $num_items_errored += $bib_items_errored; # still need to record the matched biblionumber so that the # items can be reverted - my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); + my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?"); # FIXME call SetMatchedBiblionumber instead $sth2->execute($recordid, $rowref->{'import_record_id'}); SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied'); } SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored'); } } + $schema->txn_commit; # Commit final records that may not have hit callback threshold $sth->finish(); + + if ( @biblio_ids ) { + my $indexer = Koha::SearchEngine::Indexer->new({ index => $Koha::SearchEngine::BIBLIOS_INDEX }); + $indexer->index_records( \@biblio_ids, "specialUpdate", "biblioserver" ); + } + SetImportBatchStatus($batch_id, 'imported'); return ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored); } -=head2 BatchCommitItems +=head2 _batchCommitItems ($num_items_added, $num_items_errored) = - BatchCommitItems($import_record_id, $biblionumber); + _batchCommitItems($import_record_id, $biblionumber, [$action, $biblioitemnumber]); + +Private function for batch committing item changes. We do not trigger a re-index here, that is left to the caller. =cut -sub BatchCommitItems { - my ( $import_record_id, $biblionumber, $action ) = @_; +sub _batchCommitItems { + my ( $import_record_id, $biblionumber, $action, $biblioitemnumber ) = @_; my $dbh = C4::Context->dbh; - my ( $num_items_added, $num_items_errored, $num_items_replaced ) = 0; - my $sth = $dbh->prepare(" + my $num_items_added = 0; + my $num_items_errored = 0; + my $num_items_replaced = 0; + + my $sth = $dbh->prepare( " SELECT import_items_id, import_items.marcxml, encoding FROM import_items JOIN import_records USING (import_record_id) WHERE import_record_id = ? ORDER BY import_items_id - "); + " ); $sth->bind_param( 1, $import_record_id ); $sth->execute(); + while ( my $row = $sth->fetchrow_hashref() ) { my $item_marc = MARC::Record->new_from_xml( StripNonXmlChars( $row->{'marcxml'} ), 'UTF-8', $row->{'encoding'} ); - #delete date_due subfield as to not accidentally delete item checkout due dates - my ( $MARCfield, $MARCsubfield ) = GetMarcFromKohaField( 'items.onloan', GetFrameworkCode($biblionumber) ); + # Delete date_due subfield as to not accidentally delete item checkout due dates + my ( $MARCfield, $MARCsubfield ) = GetMarcFromKohaField( 'items.onloan' ); $item_marc->field($MARCfield)->delete_subfield( code => $MARCsubfield ); - my $item = TransformMarcToKoha( $dbh, $item_marc ); + my $item = TransformMarcToKoha({ record => $item_marc, kohafields => ['items.barcode','items.itemnumber'] }); - my $duplicate_barcode = exists( $item->{'barcode'} ) && GetItemnumberFromBarcode( $item->{'barcode'} ); + my $duplicate_barcode = exists( $item->{'barcode'} ) && Koha::Items->find({ barcode => $item->{'barcode'} }); my $duplicate_itemnumber = exists( $item->{'itemnumber'} ); - my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ? WHERE import_items_id = ?"); + my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ?, import_error = ? WHERE import_items_id = ?"); if ( $action eq "replace" && $duplicate_itemnumber ) { - ModItemFromMarc( $item_marc, $biblionumber, $item->{itemnumber} ); + # Duplicate itemnumbers have precedence, that way we can update barcodes by overlaying + ModItemFromMarc( $item_marc, $biblionumber, $item->{itemnumber}, { skip_record_index => 1 } ); $updsth->bind_param( 1, 'imported' ); $updsth->bind_param( 2, $item->{itemnumber} ); - $updsth->bind_param( 3, $row->{'import_items_id'} ); + $updsth->bind_param( 3, undef ); + $updsth->bind_param( 4, $row->{'import_items_id'} ); + $updsth->execute(); + $updsth->finish(); + $num_items_replaced++; + } elsif ( $action eq "replace" && $duplicate_barcode ) { + my $itemnumber = $duplicate_barcode->itemnumber; + ModItemFromMarc( $item_marc, $biblionumber, $itemnumber, { skip_record_index => 1 } ); + $updsth->bind_param( 1, 'imported' ); + $updsth->bind_param( 2, $item->{itemnumber} ); + $updsth->bind_param( 3, undef ); + $updsth->bind_param( 4, $row->{'import_items_id'} ); $updsth->execute(); $updsth->finish(); $num_items_replaced++; } elsif ($duplicate_barcode) { $updsth->bind_param( 1, 'error' ); - $updsth->bind_param( 2, 'duplicate item barcode' ); - $updsth->bind_param( 3, $row->{'import_items_id'} ); + $updsth->bind_param( 2, undef ); + $updsth->bind_param( 3, 'duplicate item barcode' ); + $updsth->bind_param( 4, $row->{'import_items_id'} ); $updsth->execute(); $num_items_errored++; } else { - my ( $item_biblionumber, $biblioitemnumber, $itemnumber ) = AddItemFromMarc( $item_marc, $biblionumber ); - $updsth->bind_param( 1, 'imported' ); - $updsth->bind_param( 2, $itemnumber ); - $updsth->bind_param( 3, $row->{'import_items_id'} ); - $updsth->execute(); - $updsth->finish(); - $num_items_added++; + # Remove the itemnumber if it exists, we want to create a new item + my ( $itemtag, $itemsubfield ) = GetMarcFromKohaField( "items.itemnumber" ); + $item_marc->field($itemtag)->delete_subfield( code => $itemsubfield ); + + my ( $item_biblionumber, $biblioitemnumber, $itemnumber ) = AddItemFromMarc( $item_marc, $biblionumber, { biblioitemnumber => $biblioitemnumber, skip_record_index => 1 } ); + if( $itemnumber ) { + $updsth->bind_param( 1, 'imported' ); + $updsth->bind_param( 2, $itemnumber ); + $updsth->bind_param( 3, undef ); + $updsth->bind_param( 4, $row->{'import_items_id'} ); + $updsth->execute(); + $updsth->finish(); + $num_items_added++; + } } } @@ -748,6 +797,10 @@ sub BatchCommitItems { sub BatchRevertRecords { my $batch_id = shift; + my $logger = Koha::Logger->get( { category => 'C4.ImportBatch' } ); + + $logger->trace("C4::ImportBatch::BatchRevertRecords( $batch_id )"); + my $record_type; my $num_deleted = 0; my $num_errors = 0; @@ -755,7 +808,7 @@ sub BatchRevertRecords { my $num_ignored = 0; my $num_items_deleted = 0; # commit (i.e., save, all records in the batch) - SetImportBatchStatus('reverting'); + SetImportBatchStatus($batch_id, 'reverting'); my $overlay_action = GetImportBatchOverlayAction($batch_id); my $nomatch_action = GetImportBatchNoMatchAction($batch_id); my $dbh = C4::Context->dbh; @@ -789,7 +842,7 @@ sub BatchRevertRecords { $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); $error = DelBiblio($rowref->{'matched_biblionumber'}); } else { - my $deletedauthid = DelAuthority($rowref->{'matched_authid'}); + DelAuthority({ authid => $rowref->{'matched_authid'} }); } if (defined $error) { $num_errors++; @@ -802,9 +855,13 @@ sub BatchRevertRecords { my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}, $marc_type); if ($record_type eq 'biblio') { my $biblionumber = $rowref->{'matched_biblionumber'}; - my ($count, $oldbiblio) = GetBiblio($biblionumber); + my $oldbiblio = Koha::Biblios->find( $biblionumber ); + + $logger->info("C4::ImportBatch::BatchRevertRecords: Biblio record $biblionumber does not exist, restoration of this record was skipped") unless $oldbiblio; + next unless $oldbiblio; # Record has since been deleted. Deleted records should stay deleted. + $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'}); - ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'}); + ModBiblio($old_record, $biblionumber, $oldbiblio->frameworkcode); } else { my $authid = $rowref->{'matched_authid'}; ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record)); @@ -819,7 +876,7 @@ sub BatchRevertRecords { my $query; if ($record_type eq 'biblio') { # remove matched_biblionumber only if there is no 'imported' item left - $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?"; + $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?"; # FIXME Remove me $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ? AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')"; } else { $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?"; @@ -852,8 +909,8 @@ sub BatchRevertItems { $sth->bind_param(1, $import_record_id); $sth->execute(); while (my $row = $sth->fetchrow_hashref()) { - my $error = DelItemCheck($dbh, $biblionumber, $row->{'itemnumber'}); - if ($error == 1){ + my $item = Koha::Items->find($row->{itemnumber}); + if ($item->safe_delete){ my $updsth = $dbh->prepare("UPDATE import_items SET status = ? WHERE import_items_id = ?"); $updsth->bind_param(1, 'reverted'); $updsth->bind_param(2, $row->{'import_items_id'}); @@ -888,6 +945,24 @@ sub CleanBatch { SetImportBatchStatus($batch_id, 'cleaned'); } +=head2 DeleteBatch + + DeleteBatch($batch_id) + +Deletes the record from the database. This can only be done +once the batch has been cleaned. + +=cut + +sub DeleteBatch { + my $batch_id = shift; + return unless defined $batch_id; + + my $dbh = C4::Context->dbh; + my $sth = $dbh->prepare('DELETE FROM import_batches WHERE import_batch_id = ?'); + $sth->execute( $batch_id ); +} + =head2 GetAllImportBatches my $results = GetAllImportBatches(); @@ -946,9 +1021,11 @@ sub GetImportBatchRangeDesc { my ($offset, $results_per_group) = @_; my $dbh = C4::Context->dbh; - my $query = "SELECT * FROM import_batches - WHERE batch_type IN ('batch', 'webservice') - ORDER BY import_batch_id DESC"; + my $query = "SELECT b.*, p.name as profile FROM import_batches b + LEFT JOIN import_batch_profiles p + ON b.profile_id = p.id + WHERE b.batch_type IN ('batch', 'webservice') + ORDER BY b.import_batch_id DESC"; my @params; if ($results_per_group){ $query .= " LIMIT ?"; @@ -972,18 +1049,23 @@ sub GetImportBatchRangeDesc { =cut sub GetItemNumbersFromImportBatch { - my ($batch_id) = @_; - my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("SELECT itemnumber FROM import_batches,import_records,import_items WHERE import_batches.import_batch_id=import_records.import_batch_id AND import_records.import_record_id=import_items.import_record_id AND import_batches.import_batch_id=?"); - $sth->execute($batch_id); - my @items ; - while ( my ($itm) = $sth->fetchrow_array ) { - push @items, $itm; - } - return @items; + my ($batch_id) = @_; + my $dbh = C4::Context->dbh; + my $sql = q| +SELECT itemnumber FROM import_items +INNER JOIN items USING (itemnumber) +INNER JOIN import_records USING (import_record_id) +WHERE import_batch_id = ?|; + my $sth = $dbh->prepare( $sql ); + $sth->execute($batch_id); + my @items ; + while ( my ($itm) = $sth->fetchrow_array ) { + push @items, $itm; + } + return @items; } -=head2 GetNumberOfImportBatches +=head2 GetNumberOfImportBatches my $count = GetNumberOfImportBatches(); @@ -998,6 +1080,25 @@ sub GetNumberOfNonZ3950ImportBatches { return $count; } +=head2 GetImportBiblios + + my $results = GetImportBiblios($importid); + +=cut + +sub GetImportBiblios { + my ($import_record_id) = @_; + + my $dbh = C4::Context->dbh; + my $query = "SELECT * FROM import_biblios WHERE import_record_id = ?"; + return $dbh->selectall_arrayref( + $query, + { Slice => {} }, + $import_record_id + ); + +} + =head2 GetImportRecordsRange my $results = GetImportRecordsRange($batch_id, $offset, $results_per_group); @@ -1009,9 +1110,18 @@ starting at the given offset. =cut sub GetImportRecordsRange { - my ($batch_id, $offset, $results_per_group, $status) = @_; + my ( $batch_id, $offset, $results_per_group, $status, $parameters ) = @_; my $dbh = C4::Context->dbh; + + my $order_by = $parameters->{order_by} || 'import_record_id'; + ( $order_by ) = grep( { $_ eq $order_by } qw( import_record_id title status overlay_status ) ) ? $order_by : 'import_record_id'; + + my $order_by_direction = + uc( $parameters->{order_by_direction} // 'ASC' ) eq 'DESC' ? 'DESC' : 'ASC'; + + $order_by .= " $order_by_direction, authorized_heading" if $order_by eq 'title'; + my $query = "SELECT title, author, isbn, issn, authorized_heading, import_records.import_record_id, record_sequence, status, overlay_status, matched_biblionumber, matched_authid, record_type @@ -1025,7 +1135,8 @@ sub GetImportRecordsRange { $query .= " AND status=?"; push(@params,$status); } - $query.=" ORDER BY import_record_id"; + + $query.=" ORDER BY $order_by $order_by_direction"; if($results_per_group){ $query .= " LIMIT ?"; @@ -1061,6 +1172,7 @@ sub GetBestRecordMatch { WHERE import_record_matches.import_record_id = ? AND ( (import_records.record_type = 'biblio' AND biblio.biblionumber IS NOT NULL) OR (import_records.record_type = 'auth' AND auth_header.authid IS NOT NULL) ) + AND chosen = 1 ORDER BY score DESC, candidate_match_id DESC"); $sth->execute($import_record_id); my ($record_id) = $sth->fetchrow_array(); @@ -1102,6 +1214,22 @@ sub SetImportBatchStatus { } +=head2 SetMatchedBiblionumber + + SetMatchedBiblionumber($import_record_id, $biblionumber); + +=cut + +sub SetMatchedBiblionumber { + my ($import_record_id, $biblionumber) = @_; + + my $dbh = C4::Context->dbh; + $dbh->do( + q|UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?|, + undef, $biblionumber, $import_record_id + ); +} + =head2 GetImportBatchOverlayAction my $overlay_action = GetImportBatchOverlayAction($batch_id); @@ -1279,7 +1407,7 @@ sub SetImportRecordOverlayStatus { =head2 GetImportRecordStatus - my $overlay_status = GetImportRecordStatus($import_record_id); + my $status = GetImportRecordStatus($import_record_id); =cut @@ -1289,25 +1417,25 @@ sub GetImportRecordStatus { my $dbh = C4::Context->dbh; my $sth = $dbh->prepare("SELECT status FROM import_records WHERE import_record_id = ?"); $sth->execute($import_record_id); - my ($overlay_status) = $sth->fetchrow_array(); + my ($status) = $sth->fetchrow_array(); $sth->finish(); - return $overlay_status; + return $status; } =head2 SetImportRecordStatus - SetImportRecordStatus($import_record_id, $new_overlay_status); + SetImportRecordStatus($import_record_id, $new_status); =cut sub SetImportRecordStatus { - my ($import_record_id, $new_overlay_status) = @_; + my ($import_record_id, $new_status) = @_; my $dbh = C4::Context->dbh; my $sth = $dbh->prepare("UPDATE import_records SET status = ? WHERE import_record_id = ?"); - $sth->execute($new_overlay_status, $import_record_id); + $sth->execute($new_status, $import_record_id); $sth->finish(); } @@ -1325,7 +1453,8 @@ sub GetImportRecordMatches { my $dbh = C4::Context->dbh; # FIXME currently biblio only my $sth = $dbh->prepare_cached("SELECT title, author, biblionumber, - candidate_match_id, score, record_type + candidate_match_id, score, record_type, + chosen FROM import_records JOIN import_record_matches USING (import_record_id) LEFT JOIN biblio ON (biblionumber = candidate_match_id) @@ -1348,7 +1477,6 @@ sub GetImportRecordMatches { } - =head2 SetImportRecordMatches SetImportRecordMatches($import_record_id, @matches); @@ -1364,40 +1492,139 @@ sub SetImportRecordMatches { $delsth->execute($import_record_id); $delsth->finish(); - my $sth = $dbh->prepare("INSERT INTO import_record_matches (import_record_id, candidate_match_id, score) - VALUES (?, ?, ?)"); + my $sth = $dbh->prepare("INSERT INTO import_record_matches (import_record_id, candidate_match_id, score, chosen) + VALUES (?, ?, ?, ?)"); + my $chosen = 1; #The first match is defaulted to be chosen foreach my $match (@matches) { - $sth->execute($import_record_id, $match->{'record_id'}, $match->{'score'}); + $sth->execute($import_record_id, $match->{'record_id'}, $match->{'score'}, $chosen); + $chosen = 0; #After the first we do not default to other matches + } +} + +=head2 RecordsFromISO2709File + + my ($errors, $records) = C4::ImportBatch::RecordsFromISO2709File($input_file, $record_type, $encoding); + +Reads ISO2709 binary porridge from the given file and creates MARC::Record-objects out of it. + +@PARAM1, String, absolute path to the ISO2709 file. +@PARAM2, String, see stage_file.pl +@PARAM3, String, should be utf8 + +Returns two array refs. + +=cut + +sub RecordsFromISO2709File { + my ($input_file, $record_type, $encoding) = @_; + my @errors; + + my $marc_type = C4::Context->preference('marcflavour'); + $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth'); + + open my $fh, '<', $input_file or die "$0: cannot open input file $input_file: $!\n"; + my @marc_records; + $/ = "\035"; + while (<$fh>) { + s/^\s+//; + s/\s+$//; + next unless $_; # skip if record has only whitespace, as might occur + # if file includes newlines between each MARC record + my ($marc_record, $charset_guessed, $char_errors) = MarcToUTF8Record($_, $marc_type, $encoding); + push @marc_records, $marc_record; + if ($charset_guessed ne $encoding) { + push @errors, + "Unexpected charset $charset_guessed, expecting $encoding"; + } } + close $fh; + return ( \@errors, \@marc_records ); } +=head2 RecordsFromMARCXMLFile + + my ($errors, $records) = C4::ImportBatch::RecordsFromMARCXMLFile($input_file, $encoding); + +Creates MARC::Record-objects out of the given MARCXML-file. + +@PARAM1, String, absolute path to the ISO2709 file. +@PARAM2, String, should be utf8 + +Returns two array refs. + +=cut + +sub RecordsFromMARCXMLFile { + my ( $filename, $encoding ) = @_; + my $batch = MARC::File::XML->in( $filename ); + my ( @marcRecords, @errors, $record ); + do { + eval { $record = $batch->next( $encoding ); }; + if ($@) { + push @errors, $@; + } + push @marcRecords, $record if $record; + } while( $record ); + return (\@errors, \@marcRecords); +} + +=head2 RecordsFromMarcPlugin + + Converts text of input_file into array of MARC records with to_marc plugin + +=cut + +sub RecordsFromMarcPlugin { + my ($input_file, $plugin_class, $encoding) = @_; + my ( $text, @return ); + return \@return if !$input_file || !$plugin_class; + + # Read input file + open my $fh, '<', $input_file or die "$0: cannot open input file $input_file: $!\n"; + $/ = "\035"; + while (<$fh>) { + s/^\s+//; + s/\s+$//; + next unless $_; + $text .= $_; + } + close $fh; + + # Convert to large MARC blob with plugin + $text = Koha::Plugins::Handler->run({ + class => $plugin_class, + method => 'to_marc', + params => { data => $text }, + }) if $text; + + # Convert to array of MARC records + if( $text ) { + my $marc_type = C4::Context->preference('marcflavour'); + foreach my $blob ( split(/\x1D/, $text) ) { + next if $blob =~ /^\s*$/; + my ($marcrecord) = MarcToUTF8Record($blob, $marc_type, $encoding); + push @return, $marcrecord; + } + } + return \@return; +} # internal functions sub _create_import_record { - my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $z3950random, $marc_type) = @_; + my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $marc_type) = @_; my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml, - record_type, encoding, z3950random) + my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml, marcxml_old, + record_type, encoding) VALUES (?, ?, ?, ?, ?, ?, ?)"); - $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml($marc_type), - $record_type, $encoding, $z3950random); + $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml($marc_type), '', + $record_type, $encoding); my $import_record_id = $dbh->{'mysql_insertid'}; $sth->finish(); return $import_record_id; } -sub _update_import_record_marc { - my ($import_record_id, $marc_record, $marc_type) = @_; - - my $dbh = C4::Context->dbh; - my $sth = $dbh->prepare("UPDATE import_records SET marc = ?, marcxml = ? - WHERE import_record_id = ?"); - $sth->execute($marc_record->as_usmarc(), $marc_record->as_xml($marc_type), $import_record_id); - $sth->finish(); -} - sub _add_auth_fields { my ($import_record_id, $marc_record) = @_; @@ -1418,9 +1645,9 @@ sub _add_biblio_fields { my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record); my $dbh = C4::Context->dbh; # FIXME no controlnumber, originalsource - $isbn = C4::Koha::_isbn_cleanup($isbn); # FIXME C4::Koha::_isbn_cleanup should be made public + $isbn = C4::Koha::GetNormalizedISBN($isbn); my $sth = $dbh->prepare("INSERT INTO import_biblios (import_record_id, title, author, isbn, issn) VALUES (?, ?, ?, ?, ?)"); - $sth->execute($import_record_id, $title, $author, $isbn, $issn); + $sth->execute($import_record_id, $title, $author, $isbn, $issn) or die $sth->errstr; $sth->finish(); } @@ -1445,7 +1672,7 @@ sub _parse_biblio_fields { my ($marc_record) = @_; my $dbh = C4::Context->dbh; - my $bibliofields = TransformMarcToKoha($dbh, $marc_record, ''); + my $bibliofields = TransformMarcToKoha({ record => $marc_record, kohafields => ['biblio.title','biblio.author','biblioitems.isbn','biblioitems.issn'] }); return ($bibliofields->{'title'}, $bibliofields->{'author'}, $bibliofields->{'isbn'}, $bibliofields->{'issn'}); } @@ -1477,41 +1704,30 @@ sub _get_commit_action { if ($record_type eq 'biblio') { my ($bib_result, $bib_match, $item_result); - if ($overlay_status ne 'no_match') { - $bib_match = GetBestRecordMatch($import_record_id); - if ($overlay_action eq 'replace') { - $bib_result = defined($bib_match) ? 'replace' : 'create_new'; - } elsif ($overlay_action eq 'create_new') { - $bib_result = 'create_new'; - } elsif ($overlay_action eq 'ignore') { - $bib_result = 'ignore'; - } - if($item_action eq 'always_add' or $item_action eq 'add_only_for_matches'){ + $bib_match = GetBestRecordMatch($import_record_id); + if ($overlay_status ne 'no_match' && defined($bib_match)) { + + $bib_result = $overlay_action; + + if($item_action eq 'always_add' or $item_action eq 'add_only_for_matches'){ $item_result = 'create_new'; - } - elsif($item_action eq 'replace'){ - $item_result = 'replace'; - } - else { - $item_result = 'ignore'; - } + } elsif($item_action eq 'replace'){ + $item_result = 'replace'; + } else { + $item_result = 'ignore'; + } + } else { $bib_result = $nomatch_action; - $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore'; + $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore'; } return ($bib_result, $item_result, $bib_match); } else { # must be auths my ($auth_result, $auth_match); - if ($overlay_status ne 'no_match') { - $auth_match = GetBestRecordMatch($import_record_id); - if ($overlay_action eq 'replace') { - $auth_result = defined($auth_match) ? 'replace' : 'create_new'; - } elsif ($overlay_action eq 'create_new') { - $auth_result = 'create_new'; - } elsif ($overlay_action eq 'ignore') { - $auth_result = 'ignore'; - } + $auth_match = GetBestRecordMatch($import_record_id); + if ($overlay_status ne 'no_match' && defined($auth_match)) { + $auth_result = $overlay_action; } else { $auth_result = $nomatch_action; }