1 package C4::ImportBatch;
3 # Copyright (C) 2007 LibLime, 2012 C & P Bibliography Services
5 # This file is part of Koha.
7 # Koha is free software; you can redistribute it and/or modify it under the
8 # terms of the GNU General Public License as published by the Free Software
9 # Foundation; either version 2 of the License, or (at your option) any later
12 # Koha is distributed in the hope that it will be useful, but WITHOUT ANY
13 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
14 # A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License along
17 # with Koha; if not, write to the Free Software Foundation, Inc.,
18 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
28 use C4::AuthoritiesMarc;
30 use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
33 # set the version for version checking
34 $VERSION = 3.07.00.049;
41 GetImportRecordMarcXML
46 AddItemsToImportBiblio
57 GetStagedWebserviceBatches
58 GetImportBatchRangeDesc
59 GetNumberOfNonZ3950ImportBatches
61 GetItemNumbersFromImportBatch
65 GetImportBatchOverlayAction
66 SetImportBatchOverlayAction
67 GetImportBatchNoMatchAction
68 SetImportBatchNoMatchAction
69 GetImportBatchItemAction
70 SetImportBatchItemAction
73 GetImportRecordOverlayStatus
74 SetImportRecordOverlayStatus
77 GetImportRecordMatches
78 SetImportRecordMatches
84 C4::ImportBatch - manage batches of imported MARC records
92 =head2 GetZ3950BatchId
94 my $batchid = GetZ3950BatchId($z3950server);
96 Retrieves the ID of the import batch for the Z39.50
97 reservoir for the given target. If necessary,
98 creates the import batch.
102 sub GetZ3950BatchId {
103 my ($z3950server) = @_;
105 my $dbh = C4::Context->dbh;
106 my $sth = $dbh->prepare("SELECT import_batch_id FROM import_batches
107 WHERE batch_type = 'z3950'
109 $sth->execute($z3950server);
110 my $rowref = $sth->fetchrow_arrayref();
112 if (defined $rowref) {
115 my $batch_id = AddImportBatch( {
116 overlay_action => 'create_new',
117 import_status => 'staged',
118 batch_type => 'z3950',
119 file_name => $z3950server,
126 =head2 GetWebserviceBatchId
128 my $batchid = GetWebserviceBatchId();
130 Retrieves the ID of the import batch for webservice.
131 If necessary, creates the import batch.
135 my $WEBSERVICE_BASE_QRY = <<EOQ;
136 SELECT import_batch_id FROM import_batches
137 WHERE batch_type = 'webservice'
138 AND import_status = 'staged'
140 sub GetWebserviceBatchId {
143 my $dbh = C4::Context->dbh;
144 my $sql = $WEBSERVICE_BASE_QRY;
146 foreach my $field (qw(matcher_id overlay_action nomatch_action item_action)) {
147 if (my $val = $params->{$field}) {
148 $sql .= " AND $field = ?";
152 my $id = $dbh->selectrow_array($sql, undef, @args);
155 $params->{batch_type} = 'webservice';
156 $params->{import_status} = 'staged';
157 return AddImportBatch($params);
160 =head2 GetImportRecordMarc
162 my ($marcblob, $encoding) = GetImportRecordMarc($import_record_id);
166 sub GetImportRecordMarc {
167 my ($import_record_id) = @_;
169 my $dbh = C4::Context->dbh;
170 my $sth = $dbh->prepare("SELECT marc, encoding FROM import_records WHERE import_record_id = ?");
171 $sth->execute($import_record_id);
172 my ($marc, $encoding) = $sth->fetchrow();
174 return $marc, $encoding;
178 =head2 GetImportRecordMarcXML
180 my $marcxml = GetImportRecordMarcXML($import_record_id);
184 sub GetImportRecordMarcXML {
185 my ($import_record_id) = @_;
187 my $dbh = C4::Context->dbh;
188 my $sth = $dbh->prepare("SELECT marcxml FROM import_records WHERE import_record_id = ?");
189 $sth->execute($import_record_id);
190 my ($marcxml) = $sth->fetchrow();
196 =head2 AddImportBatch
198 my $batch_id = AddImportBatch($params_hash);
206 foreach (qw( matcher_id template_id branchcode
207 overlay_action nomatch_action item_action
208 import_status batch_type file_name comments record_type )) {
209 if (exists $params->{$_}) {
211 push @vals, $params->{$_};
214 my $dbh = C4::Context->dbh;
215 $dbh->do("INSERT INTO import_batches (".join( ',', @fields).")
216 VALUES (".join( ',', map '?', @fields).")",
219 return $dbh->{'mysql_insertid'};
222 =head2 GetImportBatch
224 my $row = GetImportBatch($batch_id);
226 Retrieve a hashref of an import_batches row.
233 my $dbh = C4::Context->dbh;
234 my $sth = $dbh->prepare_cached("SELECT * FROM import_batches WHERE import_batch_id = ?");
235 $sth->bind_param(1, $batch_id);
237 my $result = $sth->fetchrow_hashref;
243 =head2 AddBiblioToBatch
245 my $import_record_id = AddBiblioToBatch($batch_id, $record_sequence,
246 $marc_record, $encoding, $z3950random, $update_counts);
250 sub AddBiblioToBatch {
251 my $batch_id = shift;
252 my $record_sequence = shift;
253 my $marc_record = shift;
254 my $encoding = shift;
255 my $z3950random = shift;
256 my $update_counts = @_ ? shift : 1;
258 my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'biblio', $encoding, $z3950random, C4::Context->preference('marcflavour'));
259 _add_biblio_fields($import_record_id, $marc_record);
260 _update_batch_record_counts($batch_id) if $update_counts;
261 return $import_record_id;
264 =head2 ModBiblioInBatch
266 ModBiblioInBatch($import_record_id, $marc_record);
270 sub ModBiblioInBatch {
271 my ($import_record_id, $marc_record) = @_;
273 _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour'));
274 _update_biblio_fields($import_record_id, $marc_record);
278 =head2 AddAuthToBatch
280 my $import_record_id = AddAuthToBatch($batch_id, $record_sequence,
281 $marc_record, $encoding, $z3950random, $update_counts, [$marc_type]);
286 my $batch_id = shift;
287 my $record_sequence = shift;
288 my $marc_record = shift;
289 my $encoding = shift;
290 my $z3950random = shift;
291 my $update_counts = @_ ? shift : 1;
292 my $marc_type = shift || C4::Context->preference('marcflavour');
294 $marc_type = 'UNIMARCAUTH' if $marc_type eq 'UNIMARC';
296 my $import_record_id = _create_import_record($batch_id, $record_sequence, $marc_record, 'auth', $encoding, $z3950random, $marc_type);
297 _add_auth_fields($import_record_id, $marc_record);
298 _update_batch_record_counts($batch_id) if $update_counts;
299 return $import_record_id;
302 =head2 ModAuthInBatch
304 ModAuthInBatch($import_record_id, $marc_record);
309 my ($import_record_id, $marc_record) = @_;
311 my $marcflavour = C4::Context->preference('marcflavour');
312 _update_import_record_marc($import_record_id, $marc_record, $marcflavour eq 'UNIMARC' ? 'UNIMARCAUTH' : 'USMARC');
316 =head2 BatchStageMarcRecords
318 ($batch_id, $num_records, $num_items, @invalid_records) =
319 BatchStageMarcRecords($record_type, $encoding, $marc_records, $file_name,
320 $comments, $branch_code, $parse_items,
322 $progress_interval, $progress_callback);
326 sub BatchStageMarcRecords {
327 my $record_type = shift;
328 my $encoding = shift;
329 my $marc_records = shift;
330 my $file_name = shift;
331 my $comments = shift;
332 my $branch_code = shift;
333 my $parse_items = shift;
334 my $leave_as_staging = shift;
336 # optional callback to monitor status
338 my $progress_interval = 0;
339 my $progress_callback = undef;
341 $progress_interval = shift;
342 $progress_callback = shift;
343 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
344 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
347 my $batch_id = AddImportBatch( {
348 overlay_action => 'create_new',
349 import_status => 'staging',
350 batch_type => 'batch',
351 file_name => $file_name,
352 comments => $comments,
353 record_type => $record_type,
356 SetImportBatchItemAction($batch_id, 'always_add');
358 SetImportBatchItemAction($batch_id, 'ignore');
361 my $marc_type = C4::Context->preference('marcflavour');
362 $marc_type .= 'AUTH' if ($marc_type eq 'UNIMARC' && $record_type eq 'auth');
363 my @invalid_records = ();
366 # FIXME - for now, we're dealing only with bibs
368 foreach my $marc_blob (split(/\x1D/, $marc_records)) {
369 $marc_blob =~ s/^\s+//g;
370 $marc_blob =~ s/\s+$//g;
371 next unless $marc_blob;
373 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
374 &$progress_callback($rec_num);
376 my ($marc_record, $charset_guessed, $char_errors) =
377 MarcToUTF8Record($marc_blob, $marc_type, $encoding);
379 $encoding = $charset_guessed unless $encoding;
381 my $import_record_id;
382 if (scalar($marc_record->fields()) == 0) {
383 push @invalid_records, $marc_blob;
386 # Normalize the record so it doesn't have separated diacritics
387 SetUTF8Flag($marc_record);
390 if ($record_type eq 'biblio') {
391 $import_record_id = AddBiblioToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0);
393 my @import_items_ids = AddItemsToImportBiblio($batch_id, $import_record_id, $marc_record, 0);
394 $num_items += scalar(@import_items_ids);
396 } elsif ($record_type eq 'auth') {
397 $import_record_id = AddAuthToBatch($batch_id, $rec_num, $marc_record, $encoding, int(rand(99999)), 0, $marc_type);
401 unless ($leave_as_staging) {
402 SetImportBatchStatus($batch_id, 'staged');
404 # FIXME branch_code, number of bibs, number of items
405 _update_batch_record_counts($batch_id);
406 return ($batch_id, $num_valid, $num_items, @invalid_records);
409 =head2 AddItemsToImportBiblio
411 my @import_items_ids = AddItemsToImportBiblio($batch_id,
412 $import_record_id, $marc_record, $update_counts);
416 sub AddItemsToImportBiblio {
417 my $batch_id = shift;
418 my $import_record_id = shift;
419 my $marc_record = shift;
420 my $update_counts = @_ ? shift : 0;
422 my @import_items_ids = ();
424 my $dbh = C4::Context->dbh;
425 my ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
426 foreach my $item_field ($marc_record->field($item_tag)) {
427 my $item_marc = MARC::Record->new();
428 $item_marc->leader("00000 a "); # must set Leader/09 to 'a'
429 $item_marc->append_fields($item_field);
430 $marc_record->delete_field($item_field);
431 my $sth = $dbh->prepare_cached("INSERT INTO import_items (import_record_id, status, marcxml)
433 $sth->bind_param(1, $import_record_id);
434 $sth->bind_param(2, 'staged');
435 $sth->bind_param(3, $item_marc->as_xml());
437 push @import_items_ids, $dbh->{'mysql_insertid'};
441 if ($#import_items_ids > -1) {
442 _update_batch_record_counts($batch_id) if $update_counts;
443 _update_import_record_marc($import_record_id, $marc_record, C4::Context->preference('marcflavour'));
445 return @import_items_ids;
448 =head2 BatchFindDuplicates
450 my $num_with_matches = BatchFindDuplicates($batch_id, $matcher,
451 $max_matches, $progress_interval, $progress_callback);
453 Goes through the records loaded in the batch and attempts to
454 find duplicates for each one. Sets the matching status
455 of each record to "no_match" or "auto_match" as appropriate.
457 The $max_matches parameter is optional; if it is not supplied,
460 The $progress_interval and $progress_callback parameters are
461 optional; if both are supplied, the sub referred to by
462 $progress_callback will be invoked every $progress_interval
463 records using the number of records processed as the
468 sub BatchFindDuplicates {
469 my $batch_id = shift;
471 my $max_matches = @_ ? shift : 10;
473 # optional callback to monitor status
475 my $progress_interval = 0;
476 my $progress_callback = undef;
478 $progress_interval = shift;
479 $progress_callback = shift;
480 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
481 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
484 my $dbh = C4::Context->dbh;
486 my $sth = $dbh->prepare("SELECT import_record_id, record_type, marc
488 WHERE import_batch_id = ?");
489 $sth->execute($batch_id);
490 my $num_with_matches = 0;
492 while (my $rowref = $sth->fetchrow_hashref) {
494 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
495 &$progress_callback($rec_num);
497 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
499 if (defined $matcher) {
500 @matches = $matcher->get_matches($marc_record, $max_matches);
502 if (scalar(@matches) > 0) {
504 SetImportRecordMatches($rowref->{'import_record_id'}, @matches);
505 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'auto_match');
507 SetImportRecordMatches($rowref->{'import_record_id'}, ());
508 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'no_match');
512 return $num_with_matches;
515 =head2 BatchCommitRecords
517 my ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored) =
518 BatchCommitRecords($batch_id, $framework,
519 $progress_interval, $progress_callback);
523 sub BatchCommitRecords {
524 my $batch_id = shift;
525 my $framework = shift;
527 # optional callback to monitor status
529 my $progress_interval = 0;
530 my $progress_callback = undef;
532 $progress_interval = shift;
533 $progress_callback = shift;
534 $progress_interval = 0 unless $progress_interval =~ /^\d+$/ and $progress_interval > 0;
535 $progress_interval = 0 unless 'CODE' eq ref $progress_callback;
541 my $num_items_added = 0;
542 my $num_items_replaced = 0;
543 my $num_items_errored = 0;
545 # commit (i.e., save, all records in the batch)
546 SetImportBatchStatus('importing');
547 my $overlay_action = GetImportBatchOverlayAction($batch_id);
548 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
549 my $item_action = GetImportBatchItemAction($batch_id);
552 my $dbh = C4::Context->dbh;
553 my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marc, encoding
555 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
556 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
557 WHERE import_batch_id = ?");
558 $sth->execute($batch_id);
559 my $marcflavour = C4::Context->preference('marcflavour');
561 while (my $rowref = $sth->fetchrow_hashref) {
562 $record_type = $rowref->{'record_type'};
564 if ($progress_interval and (0 == ($rec_num % $progress_interval))) {
565 &$progress_callback($rec_num);
567 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'imported') {
573 if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
574 $marc_type = 'UNIMARCAUTH';
575 } elsif ($marcflavour eq 'UNIMARC') {
576 $marc_type = 'UNIMARC';
578 $marc_type = 'USMARC';
580 my $marc_record = MARC::Record->new_from_usmarc($rowref->{'marc'});
582 if ($record_type eq 'biblio') {
583 # remove any item tags - rely on BatchCommitItems
584 ($item_tag,$item_subfield) = &GetMarcFromKohaField("items.itemnumber",'');
585 foreach my $item_field ($marc_record->field($item_tag)) {
586 $marc_record->delete_field($item_field);
590 my ($record_result, $item_result, $record_match) =
591 _get_commit_action($overlay_action, $nomatch_action, $item_action,
592 $rowref->{'overlay_status'}, $rowref->{'import_record_id'}, $record_type);
596 if ($record_result eq 'create_new') {
598 if ($record_type eq 'biblio') {
599 my $biblioitemnumber;
600 ($recordid, $biblioitemnumber) = AddBiblio($marc_record, $framework);
601 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
602 if ($item_result eq 'create_new' || $item_result eq 'replace') {
603 my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result);
604 $num_items_added += $bib_items_added;
605 $num_items_replaced += $bib_items_replaced;
606 $num_items_errored += $bib_items_errored;
609 $recordid = AddAuthority($marc_record, undef, GuessAuthTypeCode($marc_record));
610 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
612 my $sth = $dbh->prepare_cached($query);
613 $sth->execute($recordid, $rowref->{'import_record_id'});
615 SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
616 } elsif ($record_result eq 'replace') {
618 $recordid = $record_match;
620 if ($record_type eq 'biblio') {
621 my ($count, $oldbiblio) = GetBiblio($recordid);
622 $oldxml = GetXmlBiblio($recordid);
624 # remove item fields so that they don't get
625 # added again if record is reverted
626 # FIXME: GetXmlBiblio output should not contain item info any more! So the next foreach should not be needed. Does not hurt either; may remove old 952s that should not have been there anymore.
627 my $old_marc = MARC::Record->new_from_xml(StripNonXmlChars($oldxml), 'UTF-8', $rowref->{'encoding'}, $marc_type);
628 foreach my $item_field ($old_marc->field($item_tag)) {
629 $old_marc->delete_field($item_field);
631 $oldxml = $old_marc->as_xml($marc_type);
633 ModBiblio($marc_record, $recordid, $oldbiblio->{'frameworkcode'});
634 $query = "UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?";
636 if ($item_result eq 'create_new' || $item_result eq 'replace') {
637 my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result);
638 $num_items_added += $bib_items_added;
639 $num_items_replaced += $bib_items_replaced;
640 $num_items_errored += $bib_items_errored;
643 $oldxml = GetAuthorityXML($recordid);
645 ModAuthority($recordid, $marc_record, GuessAuthTypeCode($marc_record));
646 $query = "UPDATE import_auths SET matched_authid = ? WHERE import_record_id = ?";
648 my $sth = $dbh->prepare_cached("UPDATE import_records SET marcxml_old = ? WHERE import_record_id = ?");
649 $sth->execute($oldxml, $rowref->{'import_record_id'});
651 my $sth2 = $dbh->prepare_cached($query);
652 $sth2->execute($recordid, $rowref->{'import_record_id'});
654 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
655 SetImportRecordStatus($rowref->{'import_record_id'}, 'imported');
656 } elsif ($record_result eq 'ignore') {
658 $recordid = $record_match;
659 if ($record_type eq 'biblio' and defined $recordid and $item_result eq 'create_new') {
660 my ($bib_items_added, $bib_items_replaced, $bib_items_errored) = BatchCommitItems($rowref->{'import_record_id'}, $recordid, $item_result);
661 $num_items_added += $bib_items_added;
662 $num_items_replaced += $bib_items_replaced;
663 $num_items_errored += $bib_items_errored;
664 # still need to record the matched biblionumber so that the
665 # items can be reverted
666 my $sth2 = $dbh->prepare_cached("UPDATE import_biblios SET matched_biblionumber = ? WHERE import_record_id = ?");
667 $sth2->execute($recordid, $rowref->{'import_record_id'});
668 SetImportRecordOverlayStatus($rowref->{'import_record_id'}, 'match_applied');
670 SetImportRecordStatus($rowref->{'import_record_id'}, 'ignored');
674 SetImportBatchStatus($batch_id, 'imported');
675 return ($num_added, $num_updated, $num_items_added, $num_items_replaced, $num_items_errored, $num_ignored);
678 =head2 BatchCommitItems
680 ($num_items_added, $num_items_errored) =
681 BatchCommitItems($import_record_id, $biblionumber);
685 sub BatchCommitItems {
686 my ( $import_record_id, $biblionumber, $action ) = @_;
688 my $dbh = C4::Context->dbh;
690 my ( $num_items_added, $num_items_errored, $num_items_replaced ) = 0;
691 my $sth = $dbh->prepare("
692 SELECT import_items_id, import_items.marcxml, encoding
694 JOIN import_records USING (import_record_id)
695 WHERE import_record_id = ?
696 ORDER BY import_items_id
698 $sth->bind_param( 1, $import_record_id );
700 while ( my $row = $sth->fetchrow_hashref() ) {
701 my $item_marc = MARC::Record->new_from_xml( StripNonXmlChars( $row->{'marcxml'} ), 'UTF-8', $row->{'encoding'} );
703 #delete date_due subfield as to not accidentally delete item checkout due dates
704 my ( $MARCfield, $MARCsubfield ) = GetMarcFromKohaField( 'items.onloan', GetFrameworkCode($biblionumber) );
705 $item_marc->field($MARCfield)->delete_subfield( code => $MARCsubfield );
707 my $item = TransformMarcToKoha( $dbh, $item_marc );
709 my $duplicate_barcode = exists( $item->{'barcode'} ) && GetItemnumberFromBarcode( $item->{'barcode'} );
710 my $duplicate_itemnumber = exists( $item->{'itemnumber'} );
712 my $updsth = $dbh->prepare("UPDATE import_items SET status = ?, itemnumber = ? WHERE import_items_id = ?");
713 if ( $action eq "replace" && $duplicate_itemnumber ) {
714 ModItemFromMarc( $item_marc, $biblionumber, $item->{itemnumber} );
715 $updsth->bind_param( 1, 'imported' );
716 $updsth->bind_param( 2, $item->{itemnumber} );
717 $updsth->bind_param( 3, $row->{'import_items_id'} );
720 $num_items_replaced++;
721 } elsif ($duplicate_barcode) {
722 $updsth->bind_param( 1, 'error' );
723 $updsth->bind_param( 2, 'duplicate item barcode' );
724 $updsth->bind_param( 3, $row->{'import_items_id'} );
726 $num_items_errored++;
728 my ( $item_biblionumber, $biblioitemnumber, $itemnumber ) = AddItemFromMarc( $item_marc, $biblionumber );
729 $updsth->bind_param( 1, 'imported' );
730 $updsth->bind_param( 2, $itemnumber );
731 $updsth->bind_param( 3, $row->{'import_items_id'} );
738 return ( $num_items_added, $num_items_replaced, $num_items_errored );
741 =head2 BatchRevertRecords
743 my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted,
744 $num_ignored) = BatchRevertRecords($batch_id);
748 sub BatchRevertRecords {
749 my $batch_id = shift;
754 my $num_reverted = 0;
756 my $num_items_deleted = 0;
757 # commit (i.e., save, all records in the batch)
758 SetImportBatchStatus('reverting');
759 my $overlay_action = GetImportBatchOverlayAction($batch_id);
760 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
761 my $dbh = C4::Context->dbh;
762 my $sth = $dbh->prepare("SELECT import_records.import_record_id, record_type, status, overlay_status, marcxml_old, encoding, matched_biblionumber, matched_authid
764 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
765 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
766 WHERE import_batch_id = ?");
767 $sth->execute($batch_id);
769 my $marcflavour = C4::Context->preference('marcflavour');
770 while (my $rowref = $sth->fetchrow_hashref) {
771 $record_type = $rowref->{'record_type'};
772 if ($rowref->{'status'} eq 'error' or $rowref->{'status'} eq 'reverted') {
776 if ($marcflavour eq 'UNIMARC' && $record_type eq 'auth') {
777 $marc_type = 'UNIMARCAUTH';
778 } elsif ($marcflavour eq 'UNIMARC') {
779 $marc_type = 'UNIMARC';
781 $marc_type = 'USMARC';
784 my $record_result = _get_revert_action($overlay_action, $rowref->{'overlay_status'}, $rowref->{'status'});
786 if ($record_result eq 'delete') {
788 if ($record_type eq 'biblio') {
789 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
790 $error = DelBiblio($rowref->{'matched_biblionumber'});
792 my $deletedauthid = DelAuthority($rowref->{'matched_authid'});
794 if (defined $error) {
798 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
800 } elsif ($record_result eq 'restore') {
802 my $old_record = MARC::Record->new_from_xml(StripNonXmlChars($rowref->{'marcxml_old'}), 'UTF-8', $rowref->{'encoding'}, $marc_type);
803 if ($record_type eq 'biblio') {
804 my $biblionumber = $rowref->{'matched_biblionumber'};
805 my ($count, $oldbiblio) = GetBiblio($biblionumber);
806 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
807 ModBiblio($old_record, $biblionumber, $oldbiblio->{'frameworkcode'});
809 my $authid = $rowref->{'matched_authid'};
810 ModAuthority($authid, $old_record, GuessAuthTypeCode($old_record));
812 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
813 } elsif ($record_result eq 'ignore') {
814 if ($record_type eq 'biblio') {
815 $num_items_deleted += BatchRevertItems($rowref->{'import_record_id'}, $rowref->{'matched_biblionumber'});
817 SetImportRecordStatus($rowref->{'import_record_id'}, 'reverted');
820 if ($record_type eq 'biblio') {
821 # remove matched_biblionumber only if there is no 'imported' item left
822 $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ?";
823 $query = "UPDATE import_biblios SET matched_biblionumber = NULL WHERE import_record_id = ? AND NOT EXISTS (SELECT * FROM import_items WHERE import_items.import_record_id=import_biblios.import_record_id and status='imported')";
825 $query = "UPDATE import_auths SET matched_authid = NULL WHERE import_record_id = ?";
827 my $sth2 = $dbh->prepare_cached($query);
828 $sth2->execute($rowref->{'import_record_id'});
832 SetImportBatchStatus($batch_id, 'reverted');
833 return ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored);
836 =head2 BatchRevertItems
838 my $num_items_deleted = BatchRevertItems($import_record_id, $biblionumber);
842 sub BatchRevertItems {
843 my ($import_record_id, $biblionumber) = @_;
845 my $dbh = C4::Context->dbh;
846 my $num_items_deleted = 0;
848 my $sth = $dbh->prepare_cached("SELECT import_items_id, itemnumber
850 JOIN items USING (itemnumber)
851 WHERE import_record_id = ?");
852 $sth->bind_param(1, $import_record_id);
854 while (my $row = $sth->fetchrow_hashref()) {
855 my $error = DelItemCheck($dbh, $biblionumber, $row->{'itemnumber'});
857 my $updsth = $dbh->prepare("UPDATE import_items SET status = ? WHERE import_items_id = ?");
858 $updsth->bind_param(1, 'reverted');
859 $updsth->bind_param(2, $row->{'import_items_id'});
862 $num_items_deleted++;
869 return $num_items_deleted;
874 CleanBatch($batch_id)
876 Deletes all staged records from the import batch
877 and sets the status of the batch to 'cleaned'. Note
878 that deleting a stage record does *not* affect
879 any record that has been committed to the database.
884 my $batch_id = shift;
885 return unless defined $batch_id;
887 C4::Context->dbh->do('DELETE FROM import_records WHERE import_batch_id = ?', {}, $batch_id);
888 SetImportBatchStatus($batch_id, 'cleaned');
891 =head2 GetAllImportBatches
893 my $results = GetAllImportBatches();
895 Returns a references to an array of hash references corresponding
896 to all import_batches rows (of batch_type 'batch'), sorted in
897 ascending order by import_batch_id.
901 sub GetAllImportBatches {
902 my $dbh = C4::Context->dbh;
903 my $sth = $dbh->prepare_cached("SELECT * FROM import_batches
904 WHERE batch_type IN ('batch', 'webservice')
905 ORDER BY import_batch_id ASC");
909 while (my $row = $sth->fetchrow_hashref) {
910 push @$results, $row;
916 =head2 GetStagedWebserviceBatches
918 my $batch_ids = GetStagedWebserviceBatches();
920 Returns a references to an array of batch id's
921 of batch_type 'webservice' that are not imported
925 my $PENDING_WEBSERVICE_BATCHES_QRY = <<EOQ;
926 SELECT import_batch_id FROM import_batches
927 WHERE batch_type = 'webservice'
928 AND import_status = 'staged'
930 sub GetStagedWebserviceBatches {
931 my $dbh = C4::Context->dbh;
932 return $dbh->selectcol_arrayref($PENDING_WEBSERVICE_BATCHES_QRY);
935 =head2 GetImportBatchRangeDesc
937 my $results = GetImportBatchRangeDesc($offset, $results_per_group);
939 Returns a reference to an array of hash references corresponding to
940 import_batches rows (sorted in descending order by import_batch_id)
941 start at the given offset.
945 sub GetImportBatchRangeDesc {
946 my ($offset, $results_per_group) = @_;
948 my $dbh = C4::Context->dbh;
949 my $query = "SELECT * FROM import_batches
950 WHERE batch_type IN ('batch', 'webservice')
951 ORDER BY import_batch_id DESC";
953 if ($results_per_group){
954 $query .= " LIMIT ?";
955 push(@params, $results_per_group);
958 $query .= " OFFSET ?";
959 push(@params, $offset);
961 my $sth = $dbh->prepare_cached($query);
962 $sth->execute(@params);
963 my $results = $sth->fetchall_arrayref({});
968 =head2 GetItemNumbersFromImportBatch
970 my @itemsnos = GetItemNumbersFromImportBatch($batch_id);
974 sub GetItemNumbersFromImportBatch {
976 my $dbh = C4::Context->dbh;
977 my $sth = $dbh->prepare("SELECT itemnumber FROM import_batches,import_records,import_items WHERE import_batches.import_batch_id=import_records.import_batch_id AND import_records.import_record_id=import_items.import_record_id AND import_batches.import_batch_id=?");
978 $sth->execute($batch_id);
980 while ( my ($itm) = $sth->fetchrow_array ) {
986 =head2 GetNumberOfImportBatches
988 my $count = GetNumberOfImportBatches();
992 sub GetNumberOfNonZ3950ImportBatches {
993 my $dbh = C4::Context->dbh;
994 my $sth = $dbh->prepare("SELECT COUNT(*) FROM import_batches WHERE batch_type != 'z3950'");
996 my ($count) = $sth->fetchrow_array();
1001 =head2 GetImportRecordsRange
1003 my $results = GetImportRecordsRange($batch_id, $offset, $results_per_group);
1005 Returns a reference to an array of hash references corresponding to
1006 import_biblios/import_auths/import_records rows for a given batch
1007 starting at the given offset.
1011 sub GetImportRecordsRange {
1012 my ($batch_id, $offset, $results_per_group, $status) = @_;
1014 my $dbh = C4::Context->dbh;
1015 my $query = "SELECT title, author, isbn, issn, authorized_heading, import_records.import_record_id,
1016 record_sequence, status, overlay_status,
1017 matched_biblionumber, matched_authid, record_type
1019 LEFT JOIN import_auths ON (import_records.import_record_id=import_auths.import_record_id)
1020 LEFT JOIN import_biblios ON (import_records.import_record_id=import_biblios.import_record_id)
1021 WHERE import_batch_id = ?";
1023 push(@params, $batch_id);
1025 $query .= " AND status=?";
1026 push(@params,$status);
1028 $query.=" ORDER BY import_record_id";
1030 if($results_per_group){
1031 $query .= " LIMIT ?";
1032 push(@params, $results_per_group);
1035 $query .= " OFFSET ?";
1036 push(@params, $offset);
1038 my $sth = $dbh->prepare_cached($query);
1039 $sth->execute(@params);
1040 my $results = $sth->fetchall_arrayref({});
1046 =head2 GetBestRecordMatch
1048 my $record_id = GetBestRecordMatch($import_record_id);
1052 sub GetBestRecordMatch {
1053 my ($import_record_id) = @_;
1055 my $dbh = C4::Context->dbh;
1056 my $sth = $dbh->prepare("SELECT candidate_match_id
1057 FROM import_record_matches
1058 JOIN import_records ON ( import_record_matches.import_record_id = import_records.import_record_id )
1059 LEFT JOIN biblio ON ( candidate_match_id = biblio.biblionumber )
1060 LEFT JOIN auth_header ON ( candidate_match_id = auth_header.authid )
1061 WHERE import_record_matches.import_record_id = ? AND
1062 ( (import_records.record_type = 'biblio' AND biblio.biblionumber IS NOT NULL) OR
1063 (import_records.record_type = 'auth' AND auth_header.authid IS NOT NULL) )
1064 ORDER BY score DESC, candidate_match_id DESC");
1065 $sth->execute($import_record_id);
1066 my ($record_id) = $sth->fetchrow_array();
1071 =head2 GetImportBatchStatus
1073 my $status = GetImportBatchStatus($batch_id);
1077 sub GetImportBatchStatus {
1078 my ($batch_id) = @_;
1080 my $dbh = C4::Context->dbh;
1081 my $sth = $dbh->prepare("SELECT import_status FROM import_batches WHERE import_batch_id = ?");
1082 $sth->execute($batch_id);
1083 my ($status) = $sth->fetchrow_array();
1089 =head2 SetImportBatchStatus
1091 SetImportBatchStatus($batch_id, $new_status);
1095 sub SetImportBatchStatus {
1096 my ($batch_id, $new_status) = @_;
1098 my $dbh = C4::Context->dbh;
1099 my $sth = $dbh->prepare("UPDATE import_batches SET import_status = ? WHERE import_batch_id = ?");
1100 $sth->execute($new_status, $batch_id);
1105 =head2 GetImportBatchOverlayAction
1107 my $overlay_action = GetImportBatchOverlayAction($batch_id);
1111 sub GetImportBatchOverlayAction {
1112 my ($batch_id) = @_;
1114 my $dbh = C4::Context->dbh;
1115 my $sth = $dbh->prepare("SELECT overlay_action FROM import_batches WHERE import_batch_id = ?");
1116 $sth->execute($batch_id);
1117 my ($overlay_action) = $sth->fetchrow_array();
1119 return $overlay_action;
1124 =head2 SetImportBatchOverlayAction
1126 SetImportBatchOverlayAction($batch_id, $new_overlay_action);
1130 sub SetImportBatchOverlayAction {
1131 my ($batch_id, $new_overlay_action) = @_;
1133 my $dbh = C4::Context->dbh;
1134 my $sth = $dbh->prepare("UPDATE import_batches SET overlay_action = ? WHERE import_batch_id = ?");
1135 $sth->execute($new_overlay_action, $batch_id);
1140 =head2 GetImportBatchNoMatchAction
1142 my $nomatch_action = GetImportBatchNoMatchAction($batch_id);
1146 sub GetImportBatchNoMatchAction {
1147 my ($batch_id) = @_;
1149 my $dbh = C4::Context->dbh;
1150 my $sth = $dbh->prepare("SELECT nomatch_action FROM import_batches WHERE import_batch_id = ?");
1151 $sth->execute($batch_id);
1152 my ($nomatch_action) = $sth->fetchrow_array();
1154 return $nomatch_action;
1159 =head2 SetImportBatchNoMatchAction
1161 SetImportBatchNoMatchAction($batch_id, $new_nomatch_action);
1165 sub SetImportBatchNoMatchAction {
1166 my ($batch_id, $new_nomatch_action) = @_;
1168 my $dbh = C4::Context->dbh;
1169 my $sth = $dbh->prepare("UPDATE import_batches SET nomatch_action = ? WHERE import_batch_id = ?");
1170 $sth->execute($new_nomatch_action, $batch_id);
1175 =head2 GetImportBatchItemAction
1177 my $item_action = GetImportBatchItemAction($batch_id);
1181 sub GetImportBatchItemAction {
1182 my ($batch_id) = @_;
1184 my $dbh = C4::Context->dbh;
1185 my $sth = $dbh->prepare("SELECT item_action FROM import_batches WHERE import_batch_id = ?");
1186 $sth->execute($batch_id);
1187 my ($item_action) = $sth->fetchrow_array();
1189 return $item_action;
1194 =head2 SetImportBatchItemAction
1196 SetImportBatchItemAction($batch_id, $new_item_action);
1200 sub SetImportBatchItemAction {
1201 my ($batch_id, $new_item_action) = @_;
1203 my $dbh = C4::Context->dbh;
1204 my $sth = $dbh->prepare("UPDATE import_batches SET item_action = ? WHERE import_batch_id = ?");
1205 $sth->execute($new_item_action, $batch_id);
1210 =head2 GetImportBatchMatcher
1212 my $matcher_id = GetImportBatchMatcher($batch_id);
1216 sub GetImportBatchMatcher {
1217 my ($batch_id) = @_;
1219 my $dbh = C4::Context->dbh;
1220 my $sth = $dbh->prepare("SELECT matcher_id FROM import_batches WHERE import_batch_id = ?");
1221 $sth->execute($batch_id);
1222 my ($matcher_id) = $sth->fetchrow_array();
1229 =head2 SetImportBatchMatcher
1231 SetImportBatchMatcher($batch_id, $new_matcher_id);
1235 sub SetImportBatchMatcher {
1236 my ($batch_id, $new_matcher_id) = @_;
1238 my $dbh = C4::Context->dbh;
1239 my $sth = $dbh->prepare("UPDATE import_batches SET matcher_id = ? WHERE import_batch_id = ?");
1240 $sth->execute($new_matcher_id, $batch_id);
1245 =head2 GetImportRecordOverlayStatus
1247 my $overlay_status = GetImportRecordOverlayStatus($import_record_id);
1251 sub GetImportRecordOverlayStatus {
1252 my ($import_record_id) = @_;
1254 my $dbh = C4::Context->dbh;
1255 my $sth = $dbh->prepare("SELECT overlay_status FROM import_records WHERE import_record_id = ?");
1256 $sth->execute($import_record_id);
1257 my ($overlay_status) = $sth->fetchrow_array();
1259 return $overlay_status;
1264 =head2 SetImportRecordOverlayStatus
1266 SetImportRecordOverlayStatus($import_record_id, $new_overlay_status);
1270 sub SetImportRecordOverlayStatus {
1271 my ($import_record_id, $new_overlay_status) = @_;
1273 my $dbh = C4::Context->dbh;
1274 my $sth = $dbh->prepare("UPDATE import_records SET overlay_status = ? WHERE import_record_id = ?");
1275 $sth->execute($new_overlay_status, $import_record_id);
1280 =head2 GetImportRecordStatus
1282 my $overlay_status = GetImportRecordStatus($import_record_id);
1286 sub GetImportRecordStatus {
1287 my ($import_record_id) = @_;
1289 my $dbh = C4::Context->dbh;
1290 my $sth = $dbh->prepare("SELECT status FROM import_records WHERE import_record_id = ?");
1291 $sth->execute($import_record_id);
1292 my ($overlay_status) = $sth->fetchrow_array();
1294 return $overlay_status;
1299 =head2 SetImportRecordStatus
1301 SetImportRecordStatus($import_record_id, $new_overlay_status);
1305 sub SetImportRecordStatus {
1306 my ($import_record_id, $new_overlay_status) = @_;
1308 my $dbh = C4::Context->dbh;
1309 my $sth = $dbh->prepare("UPDATE import_records SET status = ? WHERE import_record_id = ?");
1310 $sth->execute($new_overlay_status, $import_record_id);
1315 =head2 GetImportRecordMatches
1317 my $results = GetImportRecordMatches($import_record_id, $best_only);
1321 sub GetImportRecordMatches {
1322 my $import_record_id = shift;
1323 my $best_only = @_ ? shift : 0;
1325 my $dbh = C4::Context->dbh;
1326 # FIXME currently biblio only
1327 my $sth = $dbh->prepare_cached("SELECT title, author, biblionumber,
1328 candidate_match_id, score, record_type
1330 JOIN import_record_matches USING (import_record_id)
1331 LEFT JOIN biblio ON (biblionumber = candidate_match_id)
1332 WHERE import_record_id = ?
1333 ORDER BY score DESC, biblionumber DESC");
1334 $sth->bind_param(1, $import_record_id);
1337 while (my $row = $sth->fetchrow_hashref) {
1338 if ($row->{'record_type'} eq 'auth') {
1339 $row->{'authorized_heading'} = C4::AuthoritiesMarc::GetAuthorizedHeading( { authid => $row->{'candidate_match_id'} } );
1341 next if ($row->{'record_type'} eq 'biblio' && not $row->{'biblionumber'});
1342 push @$results, $row;
1352 =head2 SetImportRecordMatches
1354 SetImportRecordMatches($import_record_id, @matches);
1358 sub SetImportRecordMatches {
1359 my $import_record_id = shift;
1362 my $dbh = C4::Context->dbh;
1363 my $delsth = $dbh->prepare("DELETE FROM import_record_matches WHERE import_record_id = ?");
1364 $delsth->execute($import_record_id);
1367 my $sth = $dbh->prepare("INSERT INTO import_record_matches (import_record_id, candidate_match_id, score)
1369 foreach my $match (@matches) {
1370 $sth->execute($import_record_id, $match->{'record_id'}, $match->{'score'});
1375 # internal functions
1377 sub _create_import_record {
1378 my ($batch_id, $record_sequence, $marc_record, $record_type, $encoding, $z3950random, $marc_type) = @_;
1380 my $dbh = C4::Context->dbh;
1381 my $sth = $dbh->prepare("INSERT INTO import_records (import_batch_id, record_sequence, marc, marcxml,
1382 record_type, encoding, z3950random)
1383 VALUES (?, ?, ?, ?, ?, ?, ?)");
1384 $sth->execute($batch_id, $record_sequence, $marc_record->as_usmarc(), $marc_record->as_xml($marc_type),
1385 $record_type, $encoding, $z3950random);
1386 my $import_record_id = $dbh->{'mysql_insertid'};
1388 return $import_record_id;
1391 sub _update_import_record_marc {
1392 my ($import_record_id, $marc_record, $marc_type) = @_;
1394 my $dbh = C4::Context->dbh;
1395 my $sth = $dbh->prepare("UPDATE import_records SET marc = ?, marcxml = ?
1396 WHERE import_record_id = ?");
1397 $sth->execute($marc_record->as_usmarc(), $marc_record->as_xml($marc_type), $import_record_id);
1401 sub _add_auth_fields {
1402 my ($import_record_id, $marc_record) = @_;
1405 if ($marc_record->field('001')) {
1406 $controlnumber = $marc_record->field('001')->data();
1408 my $authorized_heading = C4::AuthoritiesMarc::GetAuthorizedHeading({ record => $marc_record });
1409 my $dbh = C4::Context->dbh;
1410 my $sth = $dbh->prepare("INSERT INTO import_auths (import_record_id, control_number, authorized_heading) VALUES (?, ?, ?)");
1411 $sth->execute($import_record_id, $controlnumber, $authorized_heading);
1415 sub _add_biblio_fields {
1416 my ($import_record_id, $marc_record) = @_;
1418 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1419 my $dbh = C4::Context->dbh;
1420 # FIXME no controlnumber, originalsource
1421 $isbn = C4::Koha::_isbn_cleanup($isbn); # FIXME C4::Koha::_isbn_cleanup should be made public
1422 my $sth = $dbh->prepare("INSERT INTO import_biblios (import_record_id, title, author, isbn, issn) VALUES (?, ?, ?, ?, ?)");
1423 $sth->execute($import_record_id, $title, $author, $isbn, $issn);
1428 sub _update_biblio_fields {
1429 my ($import_record_id, $marc_record) = @_;
1431 my ($title, $author, $isbn, $issn) = _parse_biblio_fields($marc_record);
1432 my $dbh = C4::Context->dbh;
1433 # FIXME no controlnumber, originalsource
1434 # FIXME 2 - should regularize normalization of ISBN wherever it is done
1438 my $sth = $dbh->prepare("UPDATE import_biblios SET title = ?, author = ?, isbn = ?, issn = ?
1439 WHERE import_record_id = ?");
1440 $sth->execute($title, $author, $isbn, $issn, $import_record_id);
1444 sub _parse_biblio_fields {
1445 my ($marc_record) = @_;
1447 my $dbh = C4::Context->dbh;
1448 my $bibliofields = TransformMarcToKoha($dbh, $marc_record, '');
1449 return ($bibliofields->{'title'}, $bibliofields->{'author'}, $bibliofields->{'isbn'}, $bibliofields->{'issn'});
1453 sub _update_batch_record_counts {
1454 my ($batch_id) = @_;
1456 my $dbh = C4::Context->dbh;
1457 my $sth = $dbh->prepare_cached("UPDATE import_batches SET
1461 WHERE import_batch_id = import_batches.import_batch_id),
1465 JOIN import_items USING (import_record_id)
1466 WHERE import_batch_id = import_batches.import_batch_id
1467 AND record_type = 'biblio')
1468 WHERE import_batch_id = ?");
1469 $sth->bind_param(1, $batch_id);
1474 sub _get_commit_action {
1475 my ($overlay_action, $nomatch_action, $item_action, $overlay_status, $import_record_id, $record_type) = @_;
1477 if ($record_type eq 'biblio') {
1478 my ($bib_result, $bib_match, $item_result);
1480 if ($overlay_status ne 'no_match') {
1481 $bib_match = GetBestRecordMatch($import_record_id);
1482 if ($overlay_action eq 'replace') {
1483 $bib_result = defined($bib_match) ? 'replace' : 'create_new';
1484 } elsif ($overlay_action eq 'create_new') {
1485 $bib_result = 'create_new';
1486 } elsif ($overlay_action eq 'ignore') {
1487 $bib_result = 'ignore';
1489 if($item_action eq 'always_add' or $item_action eq 'add_only_for_matches'){
1490 $item_result = 'create_new';
1492 elsif($item_action eq 'replace'){
1493 $item_result = 'replace';
1496 $item_result = 'ignore';
1499 $bib_result = $nomatch_action;
1500 $item_result = ($item_action eq 'always_add' or $item_action eq 'add_only_for_new') ? 'create_new' : 'ignore';
1502 return ($bib_result, $item_result, $bib_match);
1503 } else { # must be auths
1504 my ($auth_result, $auth_match);
1506 if ($overlay_status ne 'no_match') {
1507 $auth_match = GetBestRecordMatch($import_record_id);
1508 if ($overlay_action eq 'replace') {
1509 $auth_result = defined($auth_match) ? 'replace' : 'create_new';
1510 } elsif ($overlay_action eq 'create_new') {
1511 $auth_result = 'create_new';
1512 } elsif ($overlay_action eq 'ignore') {
1513 $auth_result = 'ignore';
1516 $auth_result = $nomatch_action;
1519 return ($auth_result, undef, $auth_match);
1524 sub _get_revert_action {
1525 my ($overlay_action, $overlay_status, $status) = @_;
1529 if ($status eq 'ignored') {
1530 $bib_result = 'ignore';
1532 if ($overlay_action eq 'create_new') {
1533 $bib_result = 'delete';
1535 $bib_result = ($overlay_status eq 'match_applied') ? 'restore' : 'delete';
1546 Koha Development Team <http://koha-community.org/>
1548 Galen Charlton <galen.charlton@liblime.com>