#!/usr/bin/perl
# Import an iso2709 file into Koha 3
-use strict;
-use warnings;
+use Modern::Perl;
#use diagnostics;
BEGIN {
# find Koha's Perl modules
use MARC::Batch;
use MARC::Charset;
+use Koha::Script;
use C4::Context;
use C4::Biblio;
use C4::Koha;
use C4::Debug;
use C4::Charset;
use C4::Items;
-use YAML;
+use C4::MarcModificationTemplates;
+
+use YAML::XS;
use Unicode::Normalize;
use Time::HiRes qw(gettimeofday);
use Getopt::Long;
use IO::File;
use Pod::Usage;
+use Koha::Biblios;
use Koha::SearchEngine;
use Koha::SearchEngine::Search;
my ($sourcetag,$sourcesubfield,$idmapfl, $dedup_barcode);
my $framework = '';
my $localcust;
+my $marc_mod_template = '';
+my $marc_mod_template_id = -1;
$|=1;
'dedupbarcode' => \$dedup_barcode,
'framework=s' => \$framework,
'custom:s' => \$localcust,
+ 'marcmodtemplate:s' => \$marc_mod_template,
);
$biblios ||= !$authorities;
$insert ||= !$update;
pod2usage( -verbose => 2 );
exit;
}
+if( $update && !( $match || $isbn_check ) ) {
+ warn "Using -update without -match or -isbn seems to be useless.\n";
+}
if(defined $localcust) { #local customize module
if(!-e $localcust) {
$localcust=\&customize if $localcust;
}
+if($marc_mod_template ne '') {
+ my @templates = GetModificationTemplates();
+ foreach my $this_template (@templates) {
+ if($this_template->{'name'} eq $marc_mod_template) {
+ if($marc_mod_template_id < 0) {
+ $marc_mod_template_id = $this_template->{'template_id'};
+ } else {
+ print "WARNING: MARC modification template name " .
+ "'$marc_mod_template' matches multiple templates. " .
+ "Please rename these templates\n";
+ exit 1;
+ }
+ }
+ }
+ if($marc_mod_template_id < 0) {
+ die "Can't located MARC modification template '$marc_mod_template'\n";
+ } else {
+ print "Records will be modified using MARC modification template: $marc_mod_template\n" if $verbose;
+ }
+}
+
my $dbh = C4::Context->dbh;
my $heading_fields=get_heading_fields();
+my $idmapfh;
if (defined $idmapfl) {
- open(IDMAP,">$idmapfl") or die "cannot open $idmapfl \n";
+ open($idmapfh, '>', $idmapfl) or die "cannot open $idmapfl \n";
}
if ((not defined $sourcesubfield) && (not defined $sourcetag)){
# Disable logging for the biblios and authorities import operation. It would unnecessarily
# slow the import
-
-# Disable the syspref cache so we can change logging settings
-C4::Context->disable_syspref_cache();
-# Save current CataloguingLog and AuthoritiesLog sysprefs values
-my $CataloguingLog = C4::Context->preference( 'CataloguingLog' );
-my $AuthoritiesLog = C4::Context->preference( 'AuthoritiesLog' );
-# Disable logging for both
-C4::Context->set_preference( 'CataloguingLog', 0 );
-C4::Context->set_preference( 'AuthoritiesLog', 0 );
+$ENV{OVERRIDE_SYSPREF_CataloguingLog} = 0;
+$ENV{OVERRIDE_SYSPREF_AuthoritiesLog} = 0;
if ($fk_off) {
$dbh->do("SET FOREIGN_KEY_CHECKS = 0");
if ($delete) {
if ($biblios){
print "deleting biblios\n";
- $dbh->do("truncate biblio");
- $dbh->do("truncate biblioitems");
- $dbh->do("truncate items");
+ $dbh->do("DELETE FROM biblio");
+ $dbh->do("ALTER TABLE biblio AUTO_INCREMENT = 1");
+ $dbh->do("DELETE FROM biblioitems");
+ $dbh->do("ALTER TABLE biblioitems AUTO_INCREMENT = 1");
+ $dbh->do("DELETE FROM items");
+ $dbh->do("ALTER TABLE items AUTO_INCREMENT = 1");
}
else {
print "deleting authorities\n";
my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
+# The definition of $searcher must be before MARC::Batch->new
+my $searcher = Koha::SearchEngine::Search->new(
+ {
+ index => (
+ $authorities
+ ? $Koha::SearchEngine::AUTHORITIES_INDEX
+ : $Koha::SearchEngine::BIBLIOS_INDEX
+ )
+ }
+);
+
print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
my $starttime = gettimeofday;
my $batch;
}
else {
( $tagid, $subfieldid ) =
- GetMarcFromKohaField( "biblio.biblionumber", $framework );
+ GetMarcFromKohaField( "biblio.biblionumber" );
$tagid||="001";
}
# the SQL query to search on isbn
my $sth_isbn = $dbh->prepare("SELECT biblionumber,biblioitemnumber FROM biblioitems WHERE isbn=?");
-$dbh->{AutoCommit} = 0;
my $loghandle;
if ($logfile){
$loghandle= IO::File->new($logfile, $writemode) ;
print $loghandle "id;operation;status\n";
}
-my $searcher = Koha::SearchEngine::Search->new(
- {
- index => (
- $authorities
- ? $Koha::SearchEngine::AUTHORITIES_INDEX
- : $Koha::SearchEngine::BIBLIOS_INDEX
- )
- }
-);
-
+my $schema = Koha::Database->schema;
+$schema->txn_begin;
RECORD: while ( ) {
my $record;
# get records
}
}
SetUTF8Flag($record);
+ if($marc_mod_template_id > 0) {
+ print "Modifying MARC\n" if $verbose;
+ ModifyRecordWithTemplate( $marc_mod_template_id, $record );
+ }
&$localcust($record) if $localcust;
my $isbn;
# remove trailing - in isbn (only for biblios, of course)
- if ($biblios && $cleanisbn) {
+ if( $biblios ) {
my $tag = $marcFlavour eq 'UNIMARC' ? '010' : '020';
my $field = $record->field($tag);
- my $isbn = $field && $field->subfield('a');
- if ( $isbn ) {
+ $isbn = $field && $field->subfield('a');
+ if ( $isbn && $cleanisbn ) {
$isbn =~ s/-//g;
$field->update('a' => $isbn);
}
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
}
$yamlhash->{$originalid}->{'subfields'} = \@subfields;
+ $yamlhash->{$originalid}->{'updated'} = 0;
}
next;
}
printlog({id=>$originalid||$id||$authid, op=>"edit",status=>"ok"}) if ($logfile);
}
}
- elsif (defined $authid) {
- ## An authid is defined but no authority in database : add
- eval { ( $authid ) = AddAuthority($record,$authid, $authtypecode) };
- if ($@){
- warn "Problem with authority $authid Cannot Add ".$@;
- printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ERROR"}) if ($logfile);
- }
- else{
- printlog({id=>$originalid||$id||$authid, op=>"insert",status=>"ok"}) if ($logfile);
- }
- }
else {
## True insert in database
eval { ( $authid ) = AddAuthority($record,"", $authtypecode) };
push @subfields, map { ( $_->[0] =~ /[a-z]/ ? $_->[1] : () ) } $field->subfields();
}
$yamlhash->{$originalid}->{'subfields'} = \@subfields;
+ $yamlhash->{$originalid}->{'updated'} = 1;
}
}
else {
if ($sourcetag < "010"){
if ($record->field($sourcetag)){
my $source = $record->field($sourcetag)->data();
- printf(IDMAP "%s|%s\n",$source,$biblionumber);
+ printf($idmapfh "%s|%s\n",$source,$biblionumber);
}
} else {
my $source=$record->subfield($sourcetag,$sourcesubfield);
- printf(IDMAP "%s|%s\n",$source,$biblionumber);
+ printf($idmapfh "%s|%s\n",$source,$biblionumber);
}
}
# create biblio, unless we already have it ( either match or isbn )
if ($biblionumber) {
- eval{$biblioitemnumber=GetBiblioData($biblionumber)->{biblioitemnumber};};
+ eval{
+ $biblioitemnumber = Koha::Biblios->find( $biblionumber )->biblioitem->biblioitemnumber;
+ };
if ($update) {
- eval { ( $biblionumber, $biblioitemnumber ) = ModBiblio( $record, $biblionumber, GetFrameworkCode($biblionumber) ) };
+ eval { ModBiblio( $record, $biblionumber, $framework ) };
if ($@) {
warn "ERROR: Edit biblio $biblionumber failed: $@\n";
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "ERROR" } ) if ($logfile);
}
} else {
if ($insert) {
- eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, '', { defer_marc_save => 1 } ) };
+ eval { ( $biblionumber, $biblioitemnumber ) = AddBiblio( $record, $framework, { defer_marc_save => 1 } ) };
if ($@) {
warn "ERROR: Adding biblio $biblionumber failed: $@\n";
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ERROR" } ) if ($logfile);
printlog( { id => $id || $originalid || $biblionumber, op => "insert", status => "ok" } ) if ($logfile);
}
} else {
+ warn "WARNING: Updating record ".($id||$originalid)." failed";
printlog( { id => $id || $originalid || $biblionumber, op => "update", status => "warning : not in database" } ) if ($logfile);
+ next RECORD;
}
}
eval { ( $itemnumbers_ref, $errors_ref ) = AddItemBatchFromMarc( $record, $biblionumber, $biblioitemnumber, '' ); };
C4::Biblio::_strip_item_fields($clone_record, '');
# This sets the marc fields if there was an error, and also calls
# defer_marc_save.
- ModBiblioMarc( $clone_record, $biblionumber, $framework );
+ ModBiblioMarc( $clone_record, $biblionumber );
if ( $error_adding ) {
warn "ERROR: Adding items to bib $biblionumber failed: $error_adding";
printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
next RECORD;
}
else{
- printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
+ printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
}
if ($dedup_barcode && grep { exists $_->{error_code} && $_->{error_code} eq 'duplicate_barcode' } @$errors_ref) {
# Find the record called 'barcode'
- my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField('items.barcode', $framework);
+ my ($tag, $sub) = C4::Biblio::GetMarcFromKohaField( 'items.barcode' );
# Now remove any items that didn't have a duplicate_barcode error,
# erase the barcodes on items that did, and re-add those items.
my %dupes;
printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ERROR"}) if ($logfile);
# if we failed because of an exception, assume that
# the MARC columns in biblioitems were not set.
- ModBiblioMarc( $record, $biblionumber, $framework );
+ ModBiblioMarc( $record, $biblionumber );
next RECORD;
} else {
- printlog({id=>$id||$originalid||$biblionumber, op=>"insert",status=>"ok"}) if ($logfile);
+ printlog({id=>$id||$originalid||$biblionumber, op=>"insertitem",status=>"ok"}) if ($logfile);
}
push @$errors_ref, @{ $more_errors };
}
}
$yamlhash->{$originalid} = $biblionumber if ($yamlfile);
}
- $dbh->commit() if (0 == $i % $commitnum);
+ if ( 0 == $i % $commitnum ) {
+ $schema->txn_commit;
+ $schema->txn_begin;
+ }
}
print $record->as_formatted()."\n" if ($verbose//0)==2;
last if $i == $number;
}
-$dbh->commit();
-$dbh->{AutoCommit} = 1;
-
+$schema->txn_commit;
if ($fk_off) {
$dbh->do("SET FOREIGN_KEY_CHECKS = 1");
}
-# Restore CataloguingLog
-C4::Context->set_preference( 'CataloguingLog', $CataloguingLog );
-# Restore AuthoritiesLog
-C4::Context->set_preference( 'AuthoritiesLog', $AuthoritiesLog );
+# Restore CataloguingLog and AuthoritiesLog
+delete $ENV{OVERRIDE_SYSPREF_CataloguingLog};
+delete $ENV{OVERRIDE_SYSPREF_AuthoritiesLog};
my $timeneeded = gettimeofday - $starttime;
print "\n$i MARC records done in $timeneeded seconds\n";
}
if ($yamlfile) {
open my $yamlfileout, q{>}, "$yamlfile" or die "cannot open $yamlfile \n";
- print $yamlfileout Dump($yamlhash);
+ print $yamlfileout YAML::XS::Dump($yamlhash);
}
exit 0;
my $string = build_simplequery($matchingpoint,$record);
push @searchstrings,$string if (length($string)>0);
}
- my $QParser;
- $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
- my $op;
- if ($QParser) {
- $op = '&&';
- } else {
- $op = 'and';
- }
+ my $op = 'and';
return join(" $op ",@searchstrings);
}
sub build_simplequery {
}
}
}
- my $QParser;
- $QParser = C4::Context->queryparser if (C4::Context->preference('UseQueryParser'));
- my $op;
- if ($QParser) {
- $op = '&&';
- } else {
- $op = 'and';
- }
+ my $op = 'and';
return join(" $op ",@searchstrings);
}
sub report_item_errors {
sub get_heading_fields{
my $headingfields;
if ($authtypes){
- $headingfields=YAML::LoadFile($authtypes);
+ $headingfields = YAML::XS::LoadFile($authtypes);
$headingfields={C4::Context->preference('marcflavour')=>$headingfields};
- $debug && warn YAML::Dump($headingfields);
+ $debug && warn YAML::XS::Dump($headingfields);
}
unless ($headingfields){
$headingfields=$dbh->selectall_hashref("SELECT auth_tag_to_report, authtypecode from auth_types",'auth_tag_to_report',{Slice=>{}});
=item B<-t, -test>
-Test mode: parses the file, saying what he would do, but doing nothing.
+Test mode: parses the file, saying what it would do, but doing nothing.
=item B<-s>
migration_tools subdirectory. You may pass an absolute file name or a file name
from the migration_tools directory.
+=item B<-marcmodtemplate>=I<TEMPLATE>
+
+This parameter allows you to specify the name of an existing MARC
+modification template to apply as the MARC records are imported (these
+templates are created in the "MARC modification templates" tool in Koha).
+If not specified, no MARC modification templates are used (default).
+
=back
=cut
projects / koha-ffzg.git / blobdiff