2 # small script that import an iso2709 file into koha 2.0
8 use MARC::File::USMARC;
9 # Uncomment the line below and use MARC::File::XML again when it works better.
11 # use MARC::File::XML;
17 use Time::HiRes qw(gettimeofday);
19 binmode(STDOUT, ":utf8");
23 my ( $input_marc_file, $number) = ('',0);
24 my ($version, $delete, $test_parameter,$char_encoding, $verbose, $commit);
27 'commit:f' => \$commit,
28 'file:s' => \$input_marc_file,
32 't' => \$test_parameter,
33 'c:s' => \$char_encoding,
37 # FIXME: Management of error conditions needed for record parsing problems
38 # and MARC8 character sets with mappings to Unicode not yet included in
39 # MARC::Charset. The real world rarity of these problems is not fully tested.
40 # Unmapped character sets will throw a warning currently and processing will
41 # continue with the error condition. A fairly trivial correction should
42 # address some record parsing and unmapped character set problems but I need
43 # time to implement a test and correction for undef subfields and revert to
44 # MARC8 if mappings are missing. -- thd
45 sub fMARC8ToUTF8($$) {
47 my ($verbose) = shift;
50 my $leader = $record->leader();
52 print "\n000 " . $leader;
55 foreach my $field ($record->fields()) {
56 if ($field->is_control_field()) {
59 my $fieldName = $field->tag();
60 my $fieldValue = $field->data();
61 $fieldValue =~ s/ /#/g;
62 print "\n" . $fieldName;
63 print ' ' . $fieldValue;
68 my $fieldName = $field->tag();
69 my $indicator1Value = $field->indicator(1);
70 my $indicator2Value = $field->indicator(2);
73 $indicator1Value =~ s/ /#/;
74 $indicator2Value =~ s/ /#/;
75 print "\n" . $fieldName . ' ' .
80 foreach my $subfield ($field->subfields()) {
81 my $subfieldName = $subfield->[0];
82 my $subfieldValue = $subfield->[1];
83 $subfieldValue = MARC::Charset::marc8_to_utf8($subfieldValue);
85 # Alas, MARC::Field::update() does not work correctly.
86 ## push (@subfieldsArray, $subfieldName, $subfieldValue);
88 push @subfieldsArray, [$subfieldName, $subfieldValue];
91 print " \$" . $subfieldName . ' ' . $subfieldValue;
96 # Alas, MARC::Field::update() does not work correctly.
98 # The first instance in the field of a of a repeated subfield
99 # overwrites the content from later instances with the content
100 # from the first instance.
101 ## $field->update(@subfieldsArray);
103 foreach my $subfieldRow(@subfieldsArray) {
104 my $subfieldName = $subfieldRow->[0];
105 $field->delete_subfields($subfieldName);
107 foreach my $subfieldRow(@subfieldsArray) {
108 $field->add_subfields(@$subfieldRow);
113 # Reading the indicator values again is not necessary.
114 # They were not converted.
115 # $indicator1Value = $field->indicator(1);
116 # $indicator2Value = $field->indicator(2);
117 # $indicator1Value =~ s/ /#/;
118 # $indicator2Value =~ s/ /#/;
119 print "\nCONVERTED TO UTF-8:\n" . $fieldName . ' ' .
122 foreach my $subfield ($field->subfields()) {
123 my $subfieldName = $subfield->[0];
124 my $subfieldValue = $subfield->[1];
125 print " \$" . $subfieldName . ' ' . $subfieldValue;
131 print "\n" if $verbose;
136 $record->encoding('UTF-8');
141 if ($version || ($input_marc_file eq '')) {
143 small script to import an iso2709 file into Koha.
145 \th : this version/help screen
146 \tfile /path/to/file/to/dump : the file to dump
147 \tv : verbose mode. 1 means "some infos", 2 means "MARC dumping"
148 \tn : the number of records to import. If missing, all the file is imported
149 \tcommit : the number of records to wait before performing a 'commit' operation
150 \tt : test mode : parses the file, saying what he would do, but doing nothing.
151 \tc : the characteristic MARC flavour. At the moment, only MARC21 and UNIMARC
152 \tsupported. MARC21 by default.
153 \td : delete EVERYTHING related to biblio in koha-DB before import :tables :
154 \t\tbiblio, \t\tbiblioitems, \t\tsubjects,\titems
155 \t\tadditionalauthors, \tbibliosubtitles, \tmarc_biblio,
156 \t\tmarc_subfield_table, \tmarc_word, \t\tmarc_blob_subfield
157 IMPORTANT : don't use this script before you've entered and checked your MARC parameters tables twice (or more!).
158 Otherwise, the import won't work correctly and you will get invalid data.
161 \t\$ export KOHA_CONF=/etc/koha.conf
162 \t\$ perl misc/migration_tools/bulkmarcimport.pl -d -commit 1000 -file /home/jmf/koha.mrc -n 3000
168 my $dbh = C4::Context->dbh;
171 print "deleting biblios\n";
172 $dbh->do("delete from biblio");
173 $dbh->do("delete from biblioitems");
174 $dbh->do("delete from items");
175 $dbh->do("delete from bibliosubject");
176 $dbh->do("delete from additionalauthors");
177 $dbh->do("delete from bibliosubtitle");
178 $dbh->do("delete from marc_biblio");
179 $dbh->do("delete from marc_subfield_table");
180 $dbh->do("delete from marc_word");
181 $dbh->do("delete from marc_blob_subfield");
183 if ($test_parameter) {
184 print "TESTING MODE ONLY\n DOING NOTHING\n===============\n";
187 $marcFlavour = 'MARC21' unless ($marcFlavour);
188 print "Characteristic MARC flavour: $marcFlavour\n" if $verbose;
189 my $starttime = gettimeofday;
190 my $batch = MARC::Batch->new( 'USMARC', $input_marc_file );
191 $batch->warnings_off();
192 $batch->strict_off();
198 $commitnum = $commit;
202 #1st of all, find item MARC tag.
203 my ($tagfield,$tagsubfield) = &MARCfind_marc_from_kohafield($dbh,"items.itemnumber",'');
204 # $dbh->do("lock tables biblio write, biblioitems write, items write, marc_biblio write, marc_subfield_table write, marc_blob_subfield write, marc_word write, marc_subfield_structure write, stopwords write");
205 while ( my $record = $batch->next() ) {
211 z3950_extended_services('commit',set_service_options('commit'));
212 print "COMMIT OPERATION SUCCESSFUL\n";
214 my $timeneeded = gettimeofday - $starttime;
215 die "$i MARC records imported in $timeneeded seconds\n";
217 # perform the commit operation ever so often
219 z3950_extended_services('commit',set_service_options('commit'));
221 print "COMMIT OPERATION SUCCESSFUL\n";
223 #now, parse the record, extract the item fields, and store them in somewhere else.
225 ## create an empty record object to populate
226 my $newRecord = MARC::Record->new();
227 $newRecord->leader($record->leader());
229 # go through each field in the existing record
230 foreach my $oldField ( $record->fields() ) {
232 # just reproduce tags < 010 in our new record
234 # Fields are not necessarily only numeric in the actual world of records
235 # nor in what I would recommend for additonal safe non-interfering local
236 # use fields. The following regular expression match is much safer than
237 # a numeric evaluation. -- thd
238 if ( $oldField->tag() =~ m/^00/ ) {
239 $newRecord->append_fields( $oldField );
243 # store our new subfield data in this list
244 my @newSubfields = ();
246 # go through each subfield code/data pair
247 foreach my $pair ( $oldField->subfields() ) {
248 #$pair->[1] =~ s/\<//g;
249 #$pair->[1] =~ s/\>//g;
250 push( @newSubfields, $pair->[0], $pair->[1] ); #char_decode($pair->[1],$char_encoding) );
253 # add the new field to our new record
254 my $newField = MARC::Field->new(
256 $oldField->indicator(1),
257 $oldField->indicator(2),
261 $newRecord->append_fields( $newField );
265 warn "$i ==>".$newRecord->as_formatted() if $verbose eq 2;
266 my @fields = $newRecord->field($tagfield);
270 foreach my $field (@fields) {
271 my $item = MARC::Record->new();
272 $item->append_fields($field);
274 $newRecord->delete_field($field);
277 print "$i : $nbitems items found\n" if $verbose;
278 # now, create biblio and items with NEWnewXX call.
279 unless ($test_parameter) {
280 my ($bibid,$oldbibitemnum) = NEWnewbiblio($dbh,$newRecord,'');
281 warn "ADDED biblio NB $bibid in DB\n" if $verbose;
282 for (my $i=0;$i<=$#items;$i++) {
283 warn "here is the biblioitemnumber $oldbibitemnum";
284 NEWnewitem($dbh,$items[$i],$bibid,$oldbibitemnum);
288 # final commit of the changes
289 z3950_extended_services('commit',set_service_options('commit'));
290 print "COMMIT OPERATION SUCCESSFUL\n";
292 my $timeneeded = gettimeofday - $starttime;
293 print "$i MARC records done in $timeneeded seconds\n";