koha-dev.rot13.org Git - koha_gimpoz/blob - misc/migration_tools/rebuild_nozebra.pl

   1 #!/usr/bin/perl
   2
   3 use C4::Context;
   4 use Getopt::Long;
   5 use C4::Biblio;
   6 use C4::AuthoritiesMarc;
   7
   8 use strict;
   9 #
  10 # script that fills the nozebra table
  11 #
  12 #
  13
  14 $|=1; # flushes output
  15
  16 # limit for database dumping
  17 my $limit;# = "LIMIT 1000";
  18 my $directory;
  19 my $skip_export;
  20 my $keep_export;
  21 my $reset;
  22 my $biblios;
  23 my $authorities;
  24 GetOptions(
  25         'd:s'      => \$directory,
  26         'reset'      => \$reset,
  27         's'        => \$skip_export,
  28         'k'        => \$keep_export,
  29         'b'        => \$biblios,
  30         'a'        => \$authorities,
  31         );
  32
  33 $directory = "export" unless $directory;
  34 my $dbh=C4::Context->dbh;
  35 $dbh->do("update systempreferences set value=1 where variable='NoZebra'");
  36 $dbh->do("CREATE TABLE `nozebra` (
  37                 `indexname` varchar(40) character set latin1 NOT NULL,
  38                 `value` varchar(250) character set latin1 NOT NULL,
  39                 `biblionumbers` longtext character set latin1 NOT NULL,
  40                 KEY `indexname` (`indexname`),
  41                 KEY `value` (`value`))
  42                 ENGINE=InnoDB DEFAULT CHARSET=utf8");
  43 $dbh->do("truncate nozebra");
  44 my $sth;
  45 $sth=$dbh->prepare("select biblionumber from biblioitems order by biblionumber $limit");
  46 $sth->execute();
  47 my $i=0;
  48 my %result;
  49
  50 my %index = (
  51     'title' => '200a,200c,200d,200e,225a,225d,225e,225f,225h,225i,225v,500*,501*,503*,510*,512*,513*,514*,515*,516*,517*,518*,519*,520*,530*,531*,532*,540*,541*,545*,604t,610t,605a',
  52     'author' =>'200f,600a,601a,604a,700a,700b,700c,700d,700a,701b,701c,701d,702a,702b,702c,702d,710a,710b,710c,710d,711a,711b,711c,711d,712a,712b,712c,712d',
  53     'isbn' => '010a',
  54     'issn' => '011a',
  55     'biblionumber' =>'0909',
  56     'itemtype' => '200b',
  57     'language' => '010a',
  58     'publisher' => '210x',
  59     'date' => '210d',
  60     'note' => '300a,301a,302a,303a,304a,305a,306az,307a,308a,309a,310a,311a,312a,313a,314a,315a,316a,317a,318a,319a,320a,321a,322a,323a,324a,325a,326a,327a,328a,330a,332a,333a,336a,337a,345a',
  61     'Koha-Auth-Number' => '6009,6019,6029,6039,6049,6059,6069,6109',
  62     'subject' => '600*,601*,606*,610*',
  63     'dewey' => '676a',
  64     'host-item' => '995a,995c',
  65     );
  66
  67 $|=1;
  68 while (my ($biblionumber) = $sth->fetchrow) {
  69     $i++;
  70     print "\r$i";
  71     my $record = GetMarcBiblio($biblionumber);
  72
  73     # get title of the record (to store the 10 first letters with the index)
  74     my $title;
  75     if (C4::Context->preference('marcflavour') eq 'UNIMARC') {
  76         $title = lc($record->subfield('200','a'));
  77     } else {
  78         $title = lc($record->subfield('245','a'));
  79     }
  80     # remove blancks comma (that could cause problem when decoding the string for CQL retrieval) and regexp specific values
  81     $title =~ s/ |,|;|\[|\]|\(|\)|\*//g;
  82     # limit to 10 char, should be enough, and limit the DB size
  83     $title = substr($title,0,10);
  84     #parse each field
  85     foreach my $field ($record->fields()) {
  86         #parse each subfield
  87         next if $field->tag <10;
  88         foreach my $subfield ($field->subfields()) {
  89             my $tag = $field->tag();
  90             my $subfieldcode = $subfield->[0];
  91             my $indexed=0;
  92             # check each index to see if the subfield is stored somewhere
  93             # otherwise, store it in __RAW__ index
  94             foreach my $key (keys %index) {
  95                 if ($index{$key} =~ /$tag\*/ or $index{$key} =~ /$tag$subfield/) {
  96                     $indexed=1;
  97                     my $line= lc $subfield->[1];
  98                     $line =~ s/-|\.|\?|,|;|!|'|\(|\)|\[|\]|{|}|"|<|>|&|\+|\*|\// /g;
  99                     foreach (split / /,$line) {
 100                         # see if the entry is already here
 101                         if ($result{$key}->{$_} =~ /$biblionumber,$title\-(\d);/) {
 102                             my $weight=$1+1;
 103                             $result{$key}->{$_} =~ s/$biblionumber,$title\-(\d);//;
 104                             $result{$key}->{$_} .= "$biblionumber,$title-$weight;";
 105                         } else {
 106                             $result{$key}->{$_}.="$biblionumber,$title-1;";
 107                         }
 108                     }
 109                 }
 110             }
 111             # the subfield is not indexed, store it in __RAW__ index anyway
 112             unless ($indexed) {
 113                 my $line= lc $subfield->[1];
 114                 $line =~ s/-|\.|\?|,|;|!|'|\(|\)|\[|\]|{|}|"|<|>|&|\+|\*|\// /g;
 115                 foreach (split / /,$line) {
 116 #                     warn $record->as_formatted."$_ =>".$title;
 117                         if ($result{__RAW__}->{$_} =~ /$biblionumber,$title\-(\d);/) {
 118                             my $weight=$1+1;
 119 #                             $weight++;
 120                             $result{__RAW__}->{$_} =~ s/$biblionumber,$title\-(\d);//;
 121                             $result{__RAW__}->{$_} .= "$biblionumber,$title-$weight;";
 122                         } else {
 123                             $result{__RAW__}->{$_}.="$biblionumber,$title-1;";
 124                         }
 125                 }
 126             }
 127         }
 128     }
 129 }
 130 my $sth = $dbh->prepare("INSERT INTO nozebra (indexname,value,biblionumbers) VALUES (?,?,?)");
 131 foreach my $key (keys %result) {
 132     foreach my $index (keys %{$result{$key}}) {
 133         $sth->execute($key,$index,$result{$key}->{$index});
 134         if (length($result{$key}->{$index}) >40000) {
 135             print length($result{$key}->{$index})." for $key / $index";
 136         }
 137     }
 138 }