2 # small script that import an iso2709 file into koha 2.0
6 # find Koha's Perl modules
7 # test carefully before changing this
9 eval { require "$FindBin::Bin/kohalib.pl" };
13 use Unicode::Normalize;
14 use MARC::File::USMARC;
20 use C4::AuthoritiesMarc;
21 use Time::HiRes qw(gettimeofday);
24 my ( $input_marc_file, $number) = ('',0);
25 my ($version, $delete, $test_parameter,$char_encoding, $verbose, $format, $commit);
28 'file:s' => \$input_marc_file,
32 't' => \$test_parameter,
33 'c:s' => \$char_encoding,
36 'commit:f' => \$commit,
39 if ($version || ($input_marc_file eq '')) {
41 small script to import an iso2709 file into Koha.
43 \th : this version/help screen
44 \tfile /path/to/file/to/dump : the file to dump
45 \tv : verbose mode. Valid modes are 1 and 2
46 \tn : the number of the record to import. If missing, the whole file is imported
47 \tt : test mode : parses the file, saying what it would do, but doing nothing.
48 \tc : the MARC flavour. At the moment, MARC21 and UNIMARC supported. MARC21 by default.
49 \td : delete EVERYTHING related to authorities in koha-DB before import
50 \tm : format, MARCXML or ISO2709
51 \tcommit : the number of records to wait before performing a 'commit' operation
52 IMPORTANT : don't use this script before you've entered and checked twice (or more) your MARC parameters tables.
53 If you fail the test, the import won't work correctly and you will get invalid datas.
55 SAMPLE : ./bulkmarcimport.pl -file /home/paul/koha.dev/local/npl -n 1
61 my $dbh = C4::Context->dbh;
64 print "deleting authorities\n";
65 $dbh->do("truncate auth_header");
67 if ($test_parameter) {
68 print "TESTING MODE ONLY\n DOING NOTHING\n===============\n";
71 my $marcFlavour = C4::Context->preference('marcflavour') || 'MARC21';
72 $char_encoding = 'MARC21' unless ($char_encoding);
73 print "CHAR : $char_encoding\n" if $verbose;
74 my $starttime = gettimeofday;
76 if ($format =~ /XML/i) {
77 # ugly hack follows -- MARC::File::XML, when used by MARC::Batch,
78 # appears to try to convert incoming XML records from MARC-8
79 # to UTF-8. Setting the BinaryEncoding key turns that off
80 # TODO: see what happens to ISO-8859-1 XML files.
81 # TODO: determine if MARC::Batch can be fixed to handle
82 # XML records properly -- it probably should be
83 # be using a proper push or pull XML parser to
84 # extract the records, not using regexes to look
85 # for <record>.*</record>.
86 $MARC::File::XML::_load_args{BinaryEncoding} = 'utf-8';
87 $batch = MARC::Batch->new( 'XML', $input_marc_file );
89 $batch = MARC::Batch->new( 'USMARC', $input_marc_file );
91 $batch->warnings_off();
95 $dbh->{AutoCommit} = 0;
101 RECORD: while ( my $record = $batch->next() ) {
104 print "\r$i" unless $i % 100;
106 if ($record->encoding() eq 'MARC-8') {
107 my ($guessed_charset, $charset_errors);
108 ($record, $guessed_charset, $charset_errors) = MarcToUTF8Record($record, $marcFlavour);
109 if ($guessed_charset eq 'failed') {
110 warn "ERROR: failed to perform character conversion for record $i\n";
115 warn "$i ==>".$record->as_formatted() if $verbose eq 2;
117 if (C4::Context->preference('marcflavour') eq 'MARC21') {
118 $authtypecode="PERSO_NAME" if ($record->field('100'));
119 $authtypecode="CORPO_NAME" if ($record->field('110'));
120 $authtypecode="MEETI_NAME" if ($record->field('111'));
121 $authtypecode="UNIF_TITLE" if ($record->field('130'));
122 $authtypecode="CHRON_TERM" if ($record->field('148'));
123 $authtypecode="TOPIC_TERM" if ($record->field('150'));
124 $authtypecode="GEOGR_NAME" if ($record->field('151'));
125 $authtypecode="GENRE/FORM" if ($record->field('155'));
126 next unless $authtypecode; # skip invalid records FIXME: far too simplistic
129 $authtypecode=substr($record->leader(),9,1);
130 $authtypecode="NP" if ($authtypecode eq 'a'); # personnes
131 $authtypecode="CO" if ($authtypecode eq 'b'); # collectivit�
132 $authtypecode="NG" if ($authtypecode eq 'c'); # g�graphique
133 $authtypecode="NM" if ($authtypecode eq 'd'); # marque
134 $authtypecode="NF" if ($authtypecode eq 'e'); # famille
135 $authtypecode="TI" if ($authtypecode eq 'f'); # Titre uniforme
136 $authtypecode="TI" if ($authtypecode eq 'h'); # auteur/titre
137 $authtypecode="MM" if ($authtypecode eq 'j'); # mot mati�e
140 unless ($test_parameter) {
141 my ($authid) = AddAuthority($record,0,$authtypecode);
142 warn "ADDED authority NB $authid in DB\n" if $verbose;
143 $dbh->commit() if (0 == $i % $commitnum);
148 my $timeneeded = gettimeofday - $starttime;
149 print "$i MARC record done in $timeneeded seconds\n";