6 use C4::AuthoritiesMarc;
10 # script that checks zebradir structure & create directories & mandatory files if needed
14 $|=1; # flushes output
16 # limit for database dumping
17 my $limit;# = "LIMIT 1";
33 $directory = "export" unless $directory;
36 my $biblioserverdir = C4::Context->zebraconfig('biblioserver')->{directory};
37 my $authorityserverdir = C4::Context->zebraconfig('authorityserver')->{directory};
39 my $kohadir = C4::Context->config('intranetdir');
40 my $dbh = C4::Context->dbh;
41 my ($biblionumbertagfield,$biblionumbertagsubfield) = &GetMarcFromKohaField("biblio.biblionumber","");
42 my ($biblioitemnumbertagfield,$biblioitemnumbertagsubfield) = &GetMarcFromKohaField("biblioitems.biblioitemnumber","");
44 print "some informations\n";
45 print "=================\n";
46 print "Zebra biblio directory =>$biblioserverdir\n";
47 print "Zebra authorities directory =>$authorityserverdir\n";
48 print "Koha directory =>$kohadir\n";
49 print "BIBLIONUMBER in : $biblionumbertagfield\$$biblionumbertagsubfield\n";
50 print "BIBLIOITEMNUMBER in : $biblioitemnumbertagfield\$$biblioitemnumbertagsubfield\n";
51 print "=================\n";
53 # creating zebra-biblios.cfg depending on system
56 # getting zebraidx directory
58 foreach (qw(/usr/local/bin/zebraidx
67 unless ($zebraidxdir) {
69 ERROR: could not find zebraidx directory
70 ERROR: Either zebra is not installed,
71 ERROR: or it's in a directory I don't checked.
72 ERROR: do a which zebraidx and edit this file to add the result you get
76 $zebraidxdir =~ s/\/bin\/.*//;
77 print "Info : zebra is in $zebraidxdir \n";
79 # getting modules directory
81 foreach (qw(/usr/local/lib/idzebra-2.0/modules/mod-grs-xml.so
82 /usr/local/lib/idzebra/modules/mod-grs-xml.so
83 /usr/lib/idzebra/modules/mod-grs-xml.so
84 /usr/lib/idzebra-2.0/modules/mod-grs-xml.so
91 unless ($modulesdir) {
93 ERROR: could not find mod-grs-xml.so directory
94 ERROR: Either zebra is not properly compiled (libxml2 is not setup and you don t have mod-grs-xml.so,
95 ERROR: or it's in a directory I don't checked.
96 ERROR: find where mod-grs-xml.so is and edit this file to add the result you get
100 $modulesdir =~ s/\/modules\/.*//;
101 print "Info: zebra modules dir : $modulesdir\n";
103 # getting tab directory
105 foreach (qw(/usr/local/share/idzebra/tab/explain.att
106 /usr/local/share/idzebra-2.0/tab/explain.att
107 /usr/share/idzebra/tab/explain.att
108 /usr/share/idzebra-2.0/tab/explain.att
117 ERROR: could not find explain.att directory
118 ERROR: Either zebra is not properly compiled,
119 ERROR: or it's in a directory I don't checked.
120 ERROR: find where explain.att is and edit this file to add the result you get
124 $tabdir =~ s/\/tab\/.*//;
125 print "Info: tab dir : $tabdir\n";
128 # AUTHORITIES creating directory structure
130 my $created_dir_or_file = 0;
132 print "====================\n";
133 print "checking directories & files for authorities\n";
134 print "====================\n";
135 unless (-d "$authorityserverdir") {
136 system("mkdir -p $authorityserverdir");
137 print "Info: created $authorityserverdir\n";
138 $created_dir_or_file++;
140 unless (-d "$authorityserverdir/lock") {
141 mkdir "$authorityserverdir/lock";
142 print "Info: created $authorityserverdir/lock\n";
143 $created_dir_or_file++;
145 unless (-d "$authorityserverdir/register") {
146 mkdir "$authorityserverdir/register";
147 print "Info: created $authorityserverdir/register\n";
148 $created_dir_or_file++;
150 unless (-d "$authorityserverdir/shadow") {
151 mkdir "$authorityserverdir/shadow";
152 print "Info: created $authorityserverdir/shadow\n";
153 $created_dir_or_file++;
155 unless (-d "$authorityserverdir/tab") {
156 mkdir "$authorityserverdir/tab";
157 print "Info: created $authorityserverdir/tab\n";
158 $created_dir_or_file++;
160 unless (-d "$authorityserverdir/key") {
161 mkdir "$authorityserverdir/key";
162 print "Info: created $authorityserverdir/key\n";
163 $created_dir_or_file++;
166 unless (-d "$authorityserverdir/etc") {
167 mkdir "$authorityserverdir/etc";
168 print "Info: created $authorityserverdir/etc\n";
169 $created_dir_or_file++;
173 # AUTHORITIES : copying mandatory files
175 # the record model, depending on marc flavour
176 unless (-f "$authorityserverdir/tab/record.abs") {
177 if (C4::Context->preference("marcflavour") eq "UNIMARC") {
178 system("cp -f $kohadir/misc/zebra/record_authorities_unimarc.abs $authorityserverdir/tab/record.abs");
179 print "Info: copied record.abs for UNIMARC\n";
181 system("cp -f $kohadir/misc/zebra/record_authorities_usmarc.abs $authorityserverdir/tab/record.abs");
182 print "Info: copied record.abs for USMARC\n";
184 $created_dir_or_file++;
186 unless (-f "$authorityserverdir/tab/sort-string-utf.chr") {
187 system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr $authorityserverdir/tab/sort-string-utf.chr");
188 print "Info: copied sort-string-utf.chr\n";
189 $created_dir_or_file++;
191 unless (-f "$authorityserverdir/tab/word-phrase-utf.chr") {
192 system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr $authorityserverdir/tab/word-phrase-utf.chr");
193 print "Info: copied word-phase-utf.chr\n";
194 $created_dir_or_file++;
196 unless (-f "$authorityserverdir/tab/auth1.att") {
197 system("cp -f $kohadir/misc/zebra/bib1_authorities.att $authorityserverdir/tab/auth1.att");
198 print "Info: copied auth1.att\n";
199 $created_dir_or_file++;
201 unless (-f "$authorityserverdir/tab/default.idx") {
202 system("cp -f $kohadir/misc/zebra/default.idx $authorityserverdir/tab/default.idx");
203 print "Info: copied default.idx\n";
204 $created_dir_or_file++;
207 unless (-f "$authorityserverdir/etc/ccl.properties") {
208 # system("cp -f $kohadir/misc/zebra/ccl.properties ".C4::Context->zebraconfig('authorityserver')->{ccl2rpn});
209 system("cp -f $kohadir/misc/zebra/ccl.properties $authorityserverdir/etc/ccl.properties");
210 print "Info: copied ccl.properties\n";
211 $created_dir_or_file++;
213 unless (-f "$authorityserverdir/etc/pqf.properties") {
214 # system("cp -f $kohadir/misc/zebra/pqf.properties ".C4::Context->zebraconfig('authorityserver')->{ccl2rpn});
215 system("cp -f $kohadir/misc/zebra/pqf.properties $authorityserverdir/etc/pqf.properties");
216 print "Info: copied pqf.properties\n";
217 $created_dir_or_file++;
221 # AUTHORITIES : copying mandatory files
223 unless (-f C4::Context->zebraconfig('authorityserver')->{config}) {
224 open ZD,">:utf8 ",C4::Context->zebraconfig('authorityserver')->{config};
226 # generated by KOHA/misc/migration_tools/rebuild_zebra.pl
227 profilePath:\${srcdir:-.}:$authorityserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
230 # Files that describe the attribute sets supported.
235 modulePath:$modulesdir/modules/
236 # Specify record type
237 iso2709.recordType:grs.marcxml.record
239 recordId: (auth1,Local-Number)
245 lockDir: $authorityserverdir/lock
250 register: $authorityserverdir/register:4G
251 shadow: $authorityserverdir/shadow:4G
253 # Temp File area for result sets
254 setTmpDir: $authorityserverdir/tmp
256 # Temp File area for index program
257 keyTmpDir: $authorityserverdir/key
259 # Approx. Memory usage during indexing
263 print "Info: creating zebra-authorities.cfg\n";
264 $created_dir_or_file++;
267 if ($created_dir_or_file) {
268 print "Info: created : $created_dir_or_file directories & files\n";
270 print "Info: file & directories OK\n";
274 # exporting authorities
277 print "====================\n";
278 print "SKIPPING authorities export\n";
279 print "====================\n";
281 print "====================\n";
282 print "exporting authorities\n";
283 print "====================\n";
284 mkdir "$directory" unless (-d $directory);
285 mkdir "$directory/authorities" unless (-d "$directory/authorities");
286 open(OUT,">:utf8","$directory/authorities/authorities.iso2709") or die $!;
287 my $dbh=C4::Context->dbh;
289 $sth=$dbh->prepare("select authid from auth_header $limit");
292 while (my ($authid) = $sth->fetchrow) {
293 my $record = GetAuthority($authid);
295 print "\r$i" unless ($i++ %100);
296 # remove leader length, that could be wrong, it will be calculated automatically by as_usmarc
297 # otherwise, if it's wron, zebra will fail miserabily (and never index what is after the failing record)
298 my $leader=$record->leader;
299 substr($leader,0,5)=' ';
300 substr($leader,10,7)='22 ';
301 $record->leader(substr($leader,0,24));
302 print OUT $record->as_usmarc();
308 # and reindexing everything
310 print "====================\n";
311 print "REINDEXING zebra\n";
312 print "====================\n";
313 system("zebraidx -c ".C4::Context->zebraconfig('authorityserver')->{config}." -g iso2709 -d authorities init") if ($reset);
314 system("zebraidx -c ".C4::Context->zebraconfig('authorityserver')->{config}." -g iso2709 -d authorities update $directory/authorities");
315 system("zebraidx -c ".C4::Context->zebraconfig('authorityserver')->{config}." -g iso2709 -d authorities commit");
317 print "skipping authorities\n";
319 #################################################################################################################
321 #################################################################################################################
324 print "====================\n";
325 print "checking directories & files for biblios\n";
326 print "====================\n";
329 # BIBLIOS : creating directory structure
331 unless (-d "$biblioserverdir") {
332 system("mkdir -p $biblioserverdir");
333 print "Info: created $biblioserverdir\n";
334 $created_dir_or_file++;
336 unless (-d "$biblioserverdir/lock") {
337 mkdir "$biblioserverdir/lock";
338 print "Info: created $biblioserverdir/lock\n";
339 $created_dir_or_file++;
341 unless (-d "$biblioserverdir/register") {
342 mkdir "$biblioserverdir/register";
343 print "Info: created $biblioserverdir/register\n";
344 $created_dir_or_file++;
346 unless (-d "$biblioserverdir/shadow") {
347 mkdir "$biblioserverdir/shadow";
348 print "Info: created $biblioserverdir/shadow\n";
349 $created_dir_or_file++;
351 unless (-d "$biblioserverdir/tab") {
352 mkdir "$biblioserverdir/tab";
353 print "Info: created $biblioserverdir/tab\n";
354 $created_dir_or_file++;
356 unless (-d "$biblioserverdir/key") {
357 mkdir "$biblioserverdir/key";
358 print "Info: created $biblioserverdir/key\n";
359 $created_dir_or_file++;
361 unless (-d "$biblioserverdir/etc") {
362 mkdir "$biblioserverdir/etc";
363 print "Info: created $biblioserverdir/etc\n";
364 $created_dir_or_file++;
368 # BIBLIOS : copying mandatory files
370 # the record model, depending on marc flavour
371 unless (-f "$biblioserverdir/tab/record.abs") {
372 if (C4::Context->preference("marcflavour") eq "UNIMARC") {
373 system("cp -f $kohadir/misc/zebra/record_biblios_unimarc.abs $biblioserverdir/tab/record.abs");
374 print "Info: copied record.abs for UNIMARC\n";
376 system("cp -f $kohadir/misc/zebra/record_biblios_usmarc.abs $biblioserverdir/tab/record.abs");
377 print "Info: copied record.abs for USMARC\n";
379 $created_dir_or_file++;
381 unless (-f "$biblioserverdir/tab/sort-string-utf.chr") {
382 system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr $biblioserverdir/tab/sort-string-utf.chr");
383 print "Info: copied sort-string-utf.chr\n";
384 $created_dir_or_file++;
386 unless (-f "$biblioserverdir/tab/word-phrase-utf.chr") {
387 system("cp -f $kohadir/misc/zebra/sort-string-utf_french.chr $biblioserverdir/tab/word-phrase-utf.chr");
388 print "Info: copied word-phase-utf.chr\n";
389 $created_dir_or_file++;
391 unless (-f "$biblioserverdir/tab/bib1.att") {
392 system("cp -f $kohadir/misc/zebra/bib1_biblios.att $biblioserverdir/tab/bib1.att");
393 print "Info: copied bib1.att\n";
394 $created_dir_or_file++;
396 unless (-f "$biblioserverdir/tab/default.idx") {
397 system("cp -f $kohadir/misc/zebra/default.idx $biblioserverdir/tab/default.idx");
398 print "Info: copied default.idx\n";
399 $created_dir_or_file++;
401 unless (-f "$biblioserverdir/etc/ccl.properties") {
402 # system("cp -f $kohadir/misc/zebra/ccl.properties ".C4::Context->zebraconfig('biblioserver')->{ccl2rpn});
403 system("cp -f $kohadir/misc/zebra/ccl.properties $biblioserverdir/etc/ccl.properties");
404 print "Info: copied ccl.properties\n";
405 $created_dir_or_file++;
407 unless (-f "$biblioserverdir/etc/pqf.properties") {
408 # system("cp -f $kohadir/misc/zebra/pqf.properties ".C4::Context->zebraconfig('biblioserver')->{ccl2rpn});
409 system("cp -f $kohadir/misc/zebra/pqf.properties $biblioserverdir/etc/pqf.properties");
410 print "Info: copied pqf.properties\n";
411 $created_dir_or_file++;
415 # BIBLIOS : copying mandatory files
417 unless (-f C4::Context->zebraconfig('biblioserver')->{config}) {
418 open ZD,">:utf8 ",C4::Context->zebraconfig('biblioserver')->{config};
420 # generated by KOHA/misc/migrtion_tools/rebuild_zebra.pl
421 profilePath:\${srcdir:-.}:$biblioserverdir/tab/:$tabdir/tab/:\${srcdir:-.}/tab/
424 # Files that describe the attribute sets supported.
429 modulePath:$modulesdir/modules/
430 # Specify record type
431 iso2709.recordType:grs.marcxml.record
433 recordId: (bib1,Local-Number)
439 lockDir: $biblioserverdir/lock
444 register: $biblioserverdir/register:4G
445 shadow: $biblioserverdir/shadow:4G
447 # Temp File area for result sets
448 setTmpDir: $biblioserverdir/tmp
450 # Temp File area for index program
451 keyTmpDir: $biblioserverdir/key
453 # Approx. Memory usage during indexing
457 print "Info: creating zebra-biblios.cfg\n";
458 $created_dir_or_file++;
461 if ($created_dir_or_file) {
462 print "Info: created : $created_dir_or_file directories & files\n";
464 print "Info: file & directories OK\n";
472 print "====================\n";
473 print "SKIPPING biblio export\n";
474 print "====================\n";
476 print "====================\n";
477 print "exporting biblios\n";
478 print "====================\n";
479 mkdir "$directory" unless (-d $directory);
480 mkdir "$directory/biblios" unless (-d "$directory/biblios");
481 open(OUT,">:utf8 ","$directory/biblios/export") or die $!;
482 my $dbh=C4::Context->dbh;
484 $sth=$dbh->prepare("select biblionumber from biblioitems where biblionumber >54000 order by biblionumber $limit");
487 while (my ($biblionumber) = $sth->fetchrow) {
488 my $record = GetMarcBiblio($biblionumber);
489 # warn $record->as_formatted;
490 # die if $record->subfield('090','9') eq 11;
492 # check that biblionumber & biblioitemnumber are stored in the MARC record, otherwise, add them & update the biblioitems.marcxml data.
493 my $record_correct=1;
494 next unless $record->field($biblionumbertagfield);
495 if ($biblionumbertagfield eq '001') {
496 unless ($record->field($biblionumbertagfield)->data()) {
499 # if the field where biblionumber is already exist, just update it, otherwise create it
500 if ($record->field($biblionumbertagfield)) {
501 $field = $record->field($biblionumbertagfield);
502 $field->update($biblionumber);
505 $newfield = MARC::Field->new( $biblionumbertagfield, $biblionumber);
506 $record->append_fields($newfield);
510 unless ($record->subfield($biblionumbertagfield,$biblionumbertagsubfield)) {
513 # if the field where biblionumber is already exist, just update it, otherwise create it
514 if ($record->field($biblionumbertagfield)) {
515 $field = $record->field($biblionumbertagfield);
516 $field->add_subfields($biblionumbertagsubfield => $biblionumber);
519 $newfield = MARC::Field->new( $biblionumbertagfield,'','', $biblionumbertagsubfield => $biblionumber);
520 $record->append_fields($newfield);
523 # warn "FIXED BIBLIONUMBER".$record->as_formatted;
525 unless ($record->subfield($biblioitemnumbertagfield,$biblioitemnumbertagsubfield)) {
527 # warn "INCORRECT BIBLIOITEMNUMBER :".$record->as_formatted;
529 # if the field where biblionumber is already exist, just update it, otherwise create it
530 if ($record->field($biblioitemnumbertagfield)) {
531 $field = $record->field($biblioitemnumbertagfield);
532 if ($biblioitemnumbertagfield <10) {
533 $field->update($biblionumber);
535 $field->add_subfields($biblioitemnumbertagsubfield => $biblionumber);
539 if ($biblioitemnumbertagfield <10) {
540 $newfield = MARC::Field->new( $biblioitemnumbertagfield, $biblionumber);
542 $newfield = MARC::Field->new( $biblioitemnumbertagfield,'','', $biblioitemnumbertagsubfield => $biblionumber);
544 $record->insert_grouped_field($newfield);
546 # warn "FIXED BIBLIOITEMNUMBER".$record->as_formatted;
548 unless ($record_correct) {
549 my $update_xml = $dbh->prepare("update biblioitems set marcxml=? where biblionumber=?");
550 warn "UPDATING $biblionumber (missing biblionumber or biblioitemnumber in MARC record : ".$record->as_xml;
551 $update_xml->execute($record->as_xml,$biblionumber);
554 print "\r$i" unless ($i++ %100);
555 # remove leader length, that could be wrong, it will be calculated automatically by as_usmarc
556 # otherwise, if it's wron, zebra will fail miserabily (and never index what is after the failing record)
557 my $leader=$record->leader;
558 substr($leader,0,5)=' ';
559 substr($leader,10,7)='22 ';
560 $record->leader(substr($leader,0,24));
561 print OUT $record->as_usmarc();
567 # and reindexing everything
569 print "====================\n";
570 print "REINDEXING zebra\n";
571 print "====================\n";
572 system("zebraidx -g iso2709 -c ".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios init") if ($reset);
573 system("zebraidx -g iso2709 -c ".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios update $directory/biblios");
574 system("zebraidx -g iso2709 -c ".C4::Context->zebraconfig('biblioserver')->{config}." -d biblios commit");
576 print "skipping biblios\n";
579 print "====================\n";
581 print "====================\n";
583 print "NOTHING cleaned : the $directory has been kept. You can re-run this script with the -s parameter if you just want to rebuild zebra after changing the record.abs or another zebra config file\n";
585 system("rm -rf $directory");
586 print "directory $directory deleted\n";