Add a new system preference 'AutomaticItemReturn' :
[koha_fer] / updater / updatedatabase
index 2325c3a..133bc54 100755 (executable)
@@ -21,8 +21,8 @@ use Getopt::Long;
 use C4::Context;
 
 use MARC::Record;
-use MARC::File::XML;
-
+use MARC::File::XML ( BinaryEncoding => 'utf8' );
 # FIXME - The user might be installing a new database, so can't rely
 # on /etc/koha.conf anyway.
 
@@ -93,21 +93,106 @@ my %requiretables = (
                                        KEY borrowernumber (borrowernumber),
                                        KEY type (type,externalid)
                                )",
+       opac_news => "(
+                               `idnew` int(10) unsigned NOT NULL auto_increment,
+                               `title` varchar(250) NOT NULL default '',
+                               `new` text NOT NULL,
+                               `lang` varchar(4) NOT NULL default '',
+                               `timestamp` timestamp NOT NULL default CURRENT_TIMESTAMP,
+                               PRIMARY KEY  (`idnew`)
+                               )",
+       repeatable_holidays => "(
+                               `id` int(11) NOT NULL auto_increment,
+                               `branchcode` varchar(4) NOT NULL default '',
+                               `weekday` smallint(6) default NULL,
+                               `day` smallint(6) default NULL,
+                               `month` smallint(6) default NULL,
+                               `title` varchar(50) NOT NULL default '',
+                               `description` text NOT NULL,
+                               PRIMARY KEY  (`id`)
+                               )",
+       special_holidays => "(
+                               `id` int(11) NOT NULL auto_increment,
+                               `branchcode` varchar(4) NOT NULL default '',
+                               `day` smallint(6) NOT NULL default '0',
+                               `month` smallint(6) NOT NULL default '0',
+                               `year` smallint(6) NOT NULL default '0',
+                               `isexception` smallint(1) NOT NULL default '1',
+                               `title` varchar(50) NOT NULL default '',
+                               `description` text NOT NULL,
+                               PRIMARY KEY  (`id`)
+                               )",
+       overduerules    =>"(`branchcode` varchar(255) NOT NULL default '',
+                                       `categorycode` char(2) NOT NULL default '',
+                                       `delay1` int(4) default '0',
+                                       `letter1` varchar(20) default NULL,
+                                       `debarred1` char(1) default '0',
+                                       `delay2` int(4) default '0',
+                                       `debarred2` char(1) default '0',
+                                       `letter2` varchar(20) default NULL,
+                                       `delay3` int(4) default '0',
+                                       `letter3` varchar(20) default NULL,
+                                       `debarred3` int(1) default '0',
+                                       PRIMARY KEY  (`branchcode`,`categorycode`)
+                                       )",
+       cities                  => "(`cityid` int auto_increment,
+                                               `city_name` char(100) NOT NULL,
+                                               `city_zipcode` char(20),
+                                               PRIMARY KEY (`cityid`)
+                                       )",
+       roadtype                        => "(`roadtypeid` int auto_increment,
+                                               `road_type` char(100) NOT NULL,
+                                               PRIMARY KEY (`roadtypeid`)
+                                       )",
+
+       labels                     => "(
+                               labelid int(11) NOT NULL auto_increment,
+                               itemnumber varchar(100) NOT NULL default '',
+                               timestamp timestamp(14) NOT NULL,
+                               PRIMARY KEY  (labelid)
+                               )",
+
+       labels_conf                => "(
+                               id int(4) NOT NULL auto_increment,
+                               barcodetype char(100) default '',
+                               title tinyint(1) default '0',
+                               isbn tinyint(1) default '0',
+                               itemtype tinyint(1) default '0',
+                               barcode tinyint(1) default '0',
+                               dewey tinyint(1) default '0',
+                               class tinyint(1) default '0',
+                               author tinyint(1) default '0',
+                               papertype char(100) default '',
+                               startrow int(2) default NULL,
+                               PRIMARY KEY  (id)
+                               )",
 
 );
 
 my %requirefields = (
-       subscription => { 'letter' => 'char(20) NULL'},
+       subscription => { 'letter' => 'char(20) NULL', 'distributedto' => 'text NULL'},
        itemtypes => { 'imageurl' => 'char(200) NULL'},
+       aqbookfund => { 'branchcode' => 'varchar(4) NULL'},
+       aqbudget => { 'branchcode' => 'varchar(4) NULL'},
+       auth_header => { 'marc' => 'BLOB NOT NULL', 'linkid' => 'BIGINT(20) NULL'},
+       auth_subfield_structure =>{ 'hidden' => 'TINYINT(3) NOT NULL UNSIGNED ZEROFILL', 'kohafield' => 'VARCHAR(45) NOT NULL', 'linkid' =>  'TINYINT(1) NOT NULL UNSIGNED', 'isurl' => 'TINYINT(1) UNSIGNED'},
+        statistics => { 'associatedborrower' => 'integer'},
 #    tablename        => { 'field' => 'fieldtype' },
 );
 
 my %dropable_table = (
+       sessionqueries  => 'sessionqueries',
+       marcrecorddone  => 'marcrecorddone',
+       users                   => 'users',
+       itemsprices             => 'itemsprices',
+       biblioanalysis  => 'biblioanalysis',
+       borexp                  => 'borexp',
 # tablename => 'tablename',
 );
 
 my %uselessfields = (
 # tablename => "field1,field2",
+       borrowers => "suburb,altstreetaddress,altsuburb,altcity,studentnumber,school,area,preferredcont,altcp",
        );
 # the other hash contains other actions that can't be done elsewhere. they are done
 # either BEFORE of AFTER everything else, depending on "when" entry (default => AFTER)
@@ -269,6 +354,106 @@ my %tabledata = (
             explanation         => 'hide marc specific datas like subfield code & indicators to library',
             type                => 'YesNo',
         },
+        {
+            uniquefieldrequired => 'variable',
+            variable            => 'NotifyBorrowerDeparture',
+            value               => '0',
+            forceupdate         => { 'explanation' => 1,
+                                     'type' => 1},
+            explanation         => 'Delay before expiry where a notice is sent when issuing',
+            type                => 'Integer',
+        },
+        {
+            uniquefieldrequired => 'variable',
+            variable            => 'OpacPasswordChange',
+            value               => '1',
+            forceupdate         => { 'explanation' => 1,
+                                     'type' => 1},
+            explanation         => 'Enable/Disable password change in OPAC (disable it when using LDAP auth)',
+            type                => 'YesNo',
+        },
+        {
+            uniquefieldrequired => 'variable',
+            variable            => 'useDaysMode',
+            value               => 'Calendar',
+            forceupdate         => { 'explanation' => 1,
+                                     'type' => 1},
+            explanation                        => 'How to calculate return dates : Calendar means holidays will be controled, Days means the return date don\'t depend on holidays',
+               type            => 'Choice',
+               options         => 'Calendar|Days'
+        },
+        {
+            uniquefieldrequired => 'variable',
+            variable            => 'borrowerMandatoryField',
+            value               => 'zipcode|surname',
+            forceupdate         => { 'explanation' => 1,
+                                     'type' => 1},
+            explanation         => 'List all mandatory fields for borrowers',
+            type                => 'free',
+        },
+        {
+            uniquefieldrequired => 'variable',
+            variable            => 'borrowerRelationship',
+            value               => 'father|mother,grand-mother',
+            forceupdate         => { 'explanation' => 1,
+                                     'type' => 1},
+            explanation         => 'The relationships between a guarantor & a guarantee (separated by | or ,)',
+            type                => 'free',
+        },
+        {
+            uniquefieldrequired => 'variable',
+            variable            => 'ReservesMaxPickUpDelay',
+            value               => '10',
+            forceupdate         => { 'explanation' => 1,
+                                     'type' => 1},
+            explanation         => 'Maximum delay to pick up a reserved document',
+            type                => 'free',
+        },
+        {
+            uniquefieldrequired => 'variable',
+            variable            => 'TransfersMaxDaysWarning',
+            value               => '3',
+            forceupdate         => { 'explanation' => 1,
+                                     'type' => 1},
+            explanation         => 'Max delay before considering the transfer has potentialy a problem',
+            type                => 'free',
+        },
+        {
+            uniquefieldrequired => 'variable',
+            variable            => 'memberofinstitution',
+            value               => '0',
+            forceupdate         => { 'explanation' => 1,
+                                     'type' => 1},
+            explanation         => 'Are your patrons members of institutions',
+            type                => 'YesNo',
+        },
+       {
+            uniquefieldrequired => 'variable',
+            variable            => 'ReadingHistory',
+            value               => '0',
+            forceupdate         => { 'explanation' => 1,
+                                     'type' => 1},
+            explanation         => 'Allow reading record info retrievable from issues and oldissues tables',
+            type                => 'YesNo',
+        },
+       {
+            uniquefieldrequired => 'variable',
+            variable            => 'IssuingInProcess',
+            value               => '0',
+            forceupdate         => { 'explanation' => 1,
+                                     'type' => 1},
+            explanation         => 'Allow no debt alert if the patron is issuing item that accumulate debt',
+            type                => 'YesNo',
+        },
+       {
+            uniquefieldrequired => 'variable',
+            variable            => 'AutomaticItemReturn',
+            value               => '1',
+            forceupdate         => { 'explanation' => 1,
+                                     'type' => 1},
+            explanation         => 'This Variable allow or not to return automaticly to his homebranch',
+            type                => 'YesNo',
+        },
     ],
 
 );
@@ -292,8 +477,569 @@ my %fielddefinitions = (
             extra   => ''
         },
     ],
+       aqbasket =>  [
+               {
+                       field   => 'booksellerid',
+                       type    => 'int(11)',
+                       null    => 'NOT NULL',
+                       key             => '',
+                       default => '1',
+                       extra   => '',
+               },
+       ],
+       aqbooksellers =>  [
+               {
+                       field   => 'listprice',
+                       type    => 'varchar(10)',
+                       null    => 'NULL',
+                       key             => '',
+                       default => '',
+                       extra   => '',
+               },
+               {
+                       field   => 'invoiceprice',
+                       type    => 'varchar(10)',
+                       null    => 'NULL',
+                       key             => '',
+                       default => '',
+                       extra   => '',
+               },
+       ],
+       issues =>  [
+               {
+                       field   => 'borrowernumber',
+                       type    => 'int(11)',
+                       null    => 'NULL', # can be null when a borrower is deleted and the foreign key rule executed
+                       key             => '',
+                       default => '',
+                       extra   => '',
+               },
+               {
+                       field   => 'itemnumber',
+                       type    => 'int(11)',
+                       null    => 'NULL', # can be null when a borrower is deleted and the foreign key rule executed
+                       key             => '',
+                       default => '',
+                       extra   => '',
+               },
+       ],
+       borrowers => [
+               {       field => 'B_email',
+                       type => 'text',
+                       null => 'NULL',
+                       after => 'B_zipcode',
+                },
+                {
+                       field => 'streetnumber', # street number (hidden if streettable table is empty)
+                       type => 'char(10)',
+                       null => 'NULL',
+                       after => 'initials',
+               },
+               {
+                       field => 'streettype', # street table, list builded from a system table
+                       type => 'char(50)',
+                       null => 'NULL',
+                       after => 'streetnumber',
+               },
+                {
+                       field => 'B_streetnumber', # street number (hidden if streettable table is empty)
+                       type => 'char(10)',
+                       null => 'NULL',
+                       after => 'fax',
+               },
+               {
+                       field => 'B_streettype', # street table, list builded from a system table
+                       type => 'char(50)',
+                       null => 'NULL',
+                       after => 'B_streetnumber',
+               },
+               {
+                       field => 'phonepro',
+                       type => 'text',
+                       null => 'NULL',
+                       after => 'fax',
+               },
+               {
+                       field => 'address2', # complement address
+                       type => 'text',
+                       null => 'NULL',
+                       after => 'address',
+               },
+               {
+                       field => 'emailpro',
+                       type => 'text',
+                       null => 'NULL',
+                       after => 'fax',
+               },
+               {
+                       field => 'contactfirstname', # contact's firstname
+                       type => 'text',
+                       null => 'NULL',
+                       after => 'contactname',
+               },
+               {
+                       field => 'contacttitle', # contact's title
+                       type => 'text',
+                       null => 'NULL',
+                       after => 'contactfirstname',
+               },
+       ],
+       
+       branches =>  [
+               {
+                       field   => 'branchip',
+                       type    => 'varchar(15)',
+                       null    => 'NULL',
+                       key             => '',
+                       default => '',
+                       extra   => '',
+               },
+               {
+                       field   => 'branchprinter',
+                       type    => 'varchar(100)',
+                       null    => 'NULL',
+                       key             => '',
+                       default => '',
+                       extra   => '',
+               },
+       ],
+       categories =>  [
+               {
+                       field   => 'category_type',
+                       type    => 'char(1)',
+                       null    => 'NOT NULL',
+                       key             => '',
+                       default => 'A',
+                       extra   => '',
+               },
+       ],
+       reserves =>  [
+               {
+                       field   => 'waitingdate',
+                       type    => 'date',
+                       null    => 'NULL',
+                       key             => '',
+                       default => '',
+                       extra   => '',
+               },
+       ],
+);
+
+my %indexes = (
+#      table => [
+#              {       indexname => 'index detail'
+#              }
+#      ],
+       shelfcontents => [
+               {       indexname => 'shelfnumber',
+                       content => 'shelfnumber',
+               },
+               {       indexname => 'itemnumber',
+                       content => 'itemnumber',
+               }
+       ],
+       bibliosubject => [
+               {       indexname => 'biblionumber',
+                       content => 'biblionumber',
+               }
+       ],
+       items => [
+               {       indexname => 'homebranch',
+                       content => 'homebranch',
+               },
+               {       indexname => 'holdingbranch',
+                       content => 'holdingbranch',
+               }
+       ],
+       aqbooksellers => [
+               {       indexname => 'PRIMARY',
+                       content => 'id',
+                       type => 'PRIMARY',
+               }
+       ],
+       aqbasket => [
+               {       indexname => 'booksellerid',
+                       content => 'booksellerid',
+               },
+       ],
+       aqorders => [
+               {       indexname => 'basketno',
+                       content => 'basketno',
+               },
+       ],
+       aqorderbreakdown => [
+               {       indexname => 'ordernumber',
+                       content => 'ordernumber',
+               },
+               {       indexname => 'bookfundid',
+                       content => 'bookfundid',
+               },
+       ],
+       currency => [
+               {       indexname => 'PRIMARY',
+                       content => 'currency',
+                       type => 'PRIMARY',
+               }
+       ],
 );
 
+my %foreign_keys = (
+#      table => [
+#              {       key => 'the key in table' (must be indexed)
+#                      foreigntable => 'the foreigntable name', # (the parent)
+#                      foreignkey => 'the foreign key column(s)' # (in the parent)
+#                      onUpdate => 'CASCADE|SET NULL|NO ACTION| RESTRICT',
+#                      onDelete => 'CASCADE|SET NULL|NO ACTION| RESTRICT',
+#              }
+#      ],
+       shelfcontents => [
+               {       key => 'shelfnumber',
+                       foreigntable => 'bookshelf',
+                       foreignkey => 'shelfnumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'itemnumber',
+                       foreigntable => 'items',
+                       foreignkey => 'itemnumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+       ],
+       # onDelete is RESTRICT on reference tables (branches, itemtype) as we don't want items to be 
+       # easily deleted, but branches/itemtype not too easy to empty...
+       biblioitems => [
+               {       key => 'biblionumber',
+                       foreigntable => 'biblio',
+                       foreignkey => 'biblionumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'itemtype',
+                       foreigntable => 'itemtypes',
+                       foreignkey => 'itemtype',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'RESTRICT',
+               },
+       ],
+       items => [
+               {       key => 'biblioitemnumber',
+                       foreigntable => 'biblioitems',
+                       foreignkey => 'biblioitemnumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'homebranch',
+                       foreigntable => 'branches',
+                       foreignkey => 'branchcode',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'RESTRICT',
+               },
+               {       key => 'holdingbranch',
+                       foreigntable => 'branches',
+                       foreignkey => 'branchcode',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'RESTRICT',
+               },
+       ],
+       additionalauthors => [
+               {       key => 'biblionumber',
+                       foreigntable => 'biblio',
+                       foreignkey => 'biblionumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+       ],
+       bibliosubject => [
+               {       key => 'biblionumber',
+                       foreigntable => 'biblio',
+                       foreignkey => 'biblionumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+       ],
+       aqbasket => [
+               {       key => 'booksellerid',
+                       foreigntable => 'aqbooksellers',
+                       foreignkey => 'id',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'RESTRICT',
+               },
+       ],
+       aqorders => [
+               {       key => 'basketno',
+                       foreigntable => 'aqbasket',
+                       foreignkey => 'basketno',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'biblionumber',
+                       foreigntable => 'biblio',
+                       foreignkey => 'biblionumber',
+                       onUpdate => 'SET NULL',
+                       onDelete => 'SET NULL',
+               },
+       ],
+       aqbooksellers => [
+               {       key => 'listprice',
+                       foreigntable => 'currency',
+                       foreignkey => 'currency',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'invoiceprice',
+                       foreigntable => 'currency',
+                       foreignkey => 'currency',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+       ],
+       aqorderbreakdown => [
+               {       key => 'ordernumber',
+                       foreigntable => 'aqorders',
+                       foreignkey => 'ordernumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'bookfundid',
+                       foreigntable => 'aqbookfund',
+                       foreignkey => 'bookfundid',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+       ],
+       branchtransfers => [
+               {       key => 'frombranch',
+                       foreigntable => 'branches',
+                       foreignkey => 'branchcode',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'tobranch',
+                       foreigntable => 'branches',
+                       foreignkey => 'branchcode',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'itemnumber',
+                       foreigntable => 'items',
+                       foreignkey => 'itemnumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+       ],
+       issuingrules => [
+               {       key => 'categorycode',
+                       foreigntable => 'categories',
+                       foreignkey => 'categorycode',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'itemtype',
+                       foreigntable => 'itemtypes',
+                       foreignkey => 'itemtype',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+       ],
+       issues => [     # constraint is SET NULL : when a borrower or an item is deleted, we keep the issuing record
+       # for stat purposes
+               {       key => 'borrowernumber',
+                       foreigntable => 'borrowers',
+                       foreignkey => 'borrowernumber',
+                       onUpdate => 'SET NULL',
+                       onDelete => 'SET NULL',
+               },
+               {       key => 'itemnumber',
+                       foreigntable => 'items',
+                       foreignkey => 'itemnumber',
+                       onUpdate => 'SET NULL',
+                       onDelete => 'SET NULL',
+               },
+       ],
+       reserves => [
+               {       key => 'borrowernumber',
+                       foreigntable => 'borrowers',
+                       foreignkey => 'borrowernumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'biblionumber',
+                       foreigntable => 'biblio',
+                       foreignkey => 'biblionumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'itemnumber',
+                       foreigntable => 'items',
+                       foreignkey => 'itemnumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'branchcode',
+                       foreigntable => 'branches',
+                       foreignkey => 'branchcode',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+       ],
+       borrowers => [ # foreign keys are RESTRICT as we don't want to delete borrowers when a branch is deleted
+       # but prevent deleting a branch as soon as it has 1 borrower !
+               {       key => 'categorycode',
+                       foreigntable => 'categories',
+                       foreignkey => 'categorycode',
+                       onUpdate => 'RESTRICT',
+                       onDelete => 'RESTRICT',
+               },
+               {       key => 'branchcode',
+                       foreigntable => 'branches',
+                       foreignkey => 'branchcode',
+                       onUpdate => 'RESTRICT',
+                       onDelete => 'RESTRICT',
+               },
+       ],
+       accountlines => [
+               {       key => 'borrowernumber',
+                       foreigntable => 'borrowers',
+                       foreignkey => 'borrowernumber',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+               {       key => 'itemnumber',
+                       foreigntable => 'items',
+                       foreignkey => 'itemnumber',
+                       onUpdate => 'SET NULL',
+                       onDelete => 'SET NULL',
+               },
+       ],
+       auth_tag_structure => [
+               {       key => 'authtypecode',
+                       foreigntable => 'auth_types',
+                       foreignkey => 'authtypecode',
+                       onUpdate => 'CASCADE',
+                       onDelete => 'CASCADE',
+               },
+       ],
+       # FIXME : don't constraint auth_*_table and auth_word, as they may be replaced by zebra
+);
+
+
+# column changes
+my %column_change = (
+       # table
+       borrowers => [
+                               {
+                                       from => 'emailaddress',
+                                       to => 'email',
+                                       after => 'city',
+                               },
+                               {
+                                       from => 'streetaddress',
+                                       to => 'address',
+                                       after => 'initials',
+                               },
+                               {
+                                       from => 'faxnumber',
+                                       to => 'fax',
+                                       after => 'phone',
+                               },
+                               {
+                                       from => 'textmessaging',
+                                       to => 'opacnote',
+                                       after => 'userid',
+                               },
+                               {
+                                       from => 'altnotes',
+                                       to => 'contactnote',
+                                       after => 'opacnote',
+                               },
+                               {
+                                       from => 'physstreet',
+                                       to => 'B_address',
+                                       after => 'fax',
+                               },
+                               {
+                                       from => 'streetcity',
+                                       to => 'B_city',
+                                       after => 'B_address',
+                               },
+                               {
+                                       from => 'phoneday',
+                                       to => 'mobile',
+                                       after => 'phone',
+                               },
+                               {
+                                       from => 'zipcode',
+                                       to => 'zipcode',
+                                       after => 'city',
+                               },
+                               {
+                                       from => 'homezipcode',
+                                       to => 'B_zipcode',
+                                       after => 'B_city',
+                               },
+                               {
+                                       from => 'altphone',
+                                       to => 'B_phone',
+                                       after => 'B_zipcode',
+                               },
+                               {
+                                       from => 'expiry',
+                                       to => 'dateexpiry',
+                                       after => 'dateenrolled',
+                               },
+                               {
+                                       from => 'guarantor',
+                                       to => 'guarantorid',
+                                       after => 'contactname',
+                               },
+                               {
+                                       from => 'textmessaging',
+                                       to => 'opacnotes',
+                                       after => 'flags',
+                               },
+                               {
+                                       from => 'altnotes',
+                                       to => 'contactnotes',
+                                       after => 'opacnotes',
+                               },
+                               {
+                                       from => 'altrelationship',
+                                       to => 'relationship',
+                                       after => 'borrowernotes',
+                               },
+                       ],
+               );
+               
+foreach my $table (keys %column_change) {
+       $sth = $dbh->prepare("show columns from $table");
+       $sth->execute();
+       undef %types;
+       while ( ( $column, $type, $null, $key, $default, $extra ) = $sth->fetchrow )
+       {
+               $types{$column}->{type} ="$type";
+               $types{$column}->{null} = "$null";
+               $types{$column}->{key} = "$key";
+               $types{$column}->{default} = "$default";
+               $types{$column}->{extra} = "$extra";
+       }    # while
+       my $tablerows = $column_change{$table};
+       foreach my $row ( @$tablerows ) {
+               if ($types{$row->{from}}->{type}) {
+                       print "altering $table $row->{from} to $row->{to}\n";
+                       # ALTER TABLE `borrowers` CHANGE `faxnumber` `fax` TEXT CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL 
+#                      alter table `borrowers` change `faxnumber` `fax` type text  null after phone
+                       my $sql = 
+                               "alter table `$table` change `$row->{from}` `$row->{to}` $types{$row->{from}}->{type} ".
+                               ($types{$row->{from}}->{null} eq 'YES'?" NULL":" NOT NULL").
+                               ($types{$row->{from}}->{default}?" default ".$types{$row->{from}}->{default}:"").
+                               "$types{$row->{from}}->{extra} after $row->{after} ";
+#                      print "$sql";
+                       $dbh->do($sql);
+               }
+       }
+}
+
 #-------------------
 # Initialize
 
@@ -399,6 +1145,7 @@ foreach $table ( keys %fielddefinitions ) {
 #              $default="''" unless $default;
                my $extra   = $row->{extra};
                my $def     = $definitions->{$field};
+               my $after       = ($row->{after}?" after ".$row->{after}:"");
 
                unless ( $type eq $def->{type}
                        && $null eq $def->{null}
@@ -425,18 +1172,48 @@ foreach $table ( keys %fielddefinitions ) {
 # if it's a primary key, drop the previous pk, before altering the table
                        my $sth;
                        if ($key ne 'PRIMARY KEY') {
-                               $sth =$dbh->prepare("alter table $table $action $field $type $null $key $extra default ?");
+                               $sth =$dbh->prepare("alter table $table $action $field $type $null $key $extra default ? $after");
                        } else {
-                               $sth =$dbh->prepare("alter table $table drop primary key, $action $field $type $null $key $extra default ?");
+                               $sth =$dbh->prepare("alter table $table drop primary key, $action $field $type $null $key $extra default ? $after");
                        }
                        $sth->execute($default);
-                       print "  Alter $field in $table\n" unless $silent;
+                       print "  alter or create $field in $table\n" unless $silent;
                }
        }
 }
 
-
 # Populate tables with required data
+
+
+# synch table and deletedtable.
+foreach my $table (('borrowers','items','biblio','biblioitems')) {
+       my %deletedborrowers;
+       print "synch'ing $table\n";
+       $sth = $dbh->prepare("show columns from deleted$table");
+       $sth->execute;
+       while ( my ( $column, $type, $null, $key, $default, $extra ) = $sth->fetchrow ) {
+               $deletedborrowers{$column}=1;
+       }
+       $sth = $dbh->prepare("show columns from $table");
+       $sth->execute;
+       my $previous;
+       while ( my ( $column, $type, $null, $key, $default, $extra ) = $sth->fetchrow ) {
+               unless ($deletedborrowers{$column}) {
+                       my $newcol="alter table deleted$table add $column $type";
+                       if ($null eq 'YES') {
+                               $newcol .= " NULL ";
+                       } else {
+                               $newcol .= " NOT NULL ";
+                       }
+                       $newcol .= "default $default" if $default;
+                       $newcol .= " after $previous" if $previous;
+                       $previous=$column;
+                       print "creating column $column\n";
+                       $dbh->do($newcol);
+               }
+       }
+}
+
 foreach my $table ( keys %tabledata ) {
     print "Checking for data required in table $table...\n" unless $silent;
     my $tablerows = $tabledata{$table};
@@ -449,36 +1226,113 @@ foreach my $table ( keys %tabledata ) {
 "select $uniquefieldrequired from $table where $uniquefieldrequired=?"
         );
         $sth->execute($uniquevalue);
-       if ($sth->rows) {
-           foreach my $field (keys %$forceupdate) {
-               if ($forceupdate->{$field}) {
-                   my $sth=$dbh->prepare("update systempreferences set $field=? where $uniquefieldrequired=?");
-                   $sth->execute($row->{$field}, $uniquevalue);
+               if ($sth->rows) {
+                       foreach my $field (keys %$forceupdate) {
+                               if ($forceupdate->{$field}) {
+                                       my $sth=$dbh->prepare("update systempreferences set $field=? where $uniquefieldrequired=?");
+                                       $sth->execute($row->{$field}, $uniquevalue);
+                               }
+               }
+               } else {
+                       print "Adding row to $table: " unless $silent;
+                       my @values;
+                       my $fieldlist;
+                       my $placeholders;
+                       foreach my $field ( keys %$row ) {
+                               next if $field eq 'uniquefieldrequired';
+                               next if $field eq 'forceupdate';
+                               my $value = $row->{$field};
+                               push @values, $value;
+                               print "  $field => $value" unless $silent;
+                               $fieldlist .= "$field,";
+                               $placeholders .= "?,";
+                       }
+                       print "\n" unless $silent;
+                       $fieldlist    =~ s/,$//;
+                       $placeholders =~ s/,$//;
+                       my $sth =
+                       $dbh->prepare(
+                               "insert into $table ($fieldlist) values ($placeholders)");
+                       $sth->execute(@values);
                }
-           }
-       } else {
-            print "Adding row to $table: " unless $silent;
-            my @values;
-            my $fieldlist;
-            my $placeholders;
-            foreach my $field ( keys %$row ) {
-                next if $field eq 'uniquefieldrequired';
-               next if $field eq 'forceupdate';
-                my $value = $row->{$field};
-                push @values, $value;
-                print "  $field => $value" unless $silent;
-                $fieldlist .= "$field,";
-                $placeholders .= "?,";
-            }
-            print "\n" unless $silent;
-            $fieldlist    =~ s/,$//;
-            $placeholders =~ s/,$//;
-            my $sth =
-              $dbh->prepare(
-                "insert into $table ($fieldlist) values ($placeholders)");
-            $sth->execute(@values);
-        }
-    }
+       }
+}
+
+#
+# check indexes and create them when needed
+#
+print "Checking for index required...\n" unless $silent;
+foreach my $table ( keys %indexes ) {
+       #
+       # read all indexes from $table
+       #
+       $sth = $dbh->prepare("show index from $table");
+       $sth->execute;
+       my %existingindexes;
+       while ( my ( $table, $non_unique, $key_name, $Seq_in_index, $Column_name, $Collation, $cardinality, $sub_part, $Packed, $comment ) = $sth->fetchrow ) {
+               $existingindexes{$key_name} = 1;
+       }
+       # read indexes to check
+       my $tablerows = $indexes{$table};
+       foreach my $row (@$tablerows) {
+               my $key_name=$row->{indexname};
+               if ($existingindexes{$key_name} eq 1) {
+#                      print "$key_name existing";
+               } else {
+                       print "\tCreating index $key_name in $table\n";
+                       my $sql;
+                       if ($row->{indexname} eq 'PRIMARY') {
+                               $sql = "alter table $table ADD PRIMARY KEY ($row->{content})";
+                       } else {
+                               $sql = "alter table $table ADD INDEX $key_name ($row->{content}) $row->{type}";
+                       }
+                       $dbh->do($sql);
+            print "Error $sql : $dbh->err \n" if $dbh->err;
+               }
+       }
+}
+
+#
+# check foreign keys and create them when needed
+#
+print "Checking for foreign keys required...\n" unless $silent;
+foreach my $table ( keys %foreign_keys ) {
+       #
+       # read all indexes from $table
+       #
+       $sth = $dbh->prepare("show table status like '$table'");
+       $sth->execute;
+       my $stat = $sth->fetchrow_hashref;
+       # read indexes to check
+       my $tablerows = $foreign_keys{$table};
+       foreach my $row (@$tablerows) {
+               my $foreign_table=$row->{foreigntable};
+               if ($stat->{'Comment'} =~/$foreign_table/) {
+#                      print "$foreign_table existing\n";
+               } else {
+                       print "\tCreating foreign key $foreign_table in $table\n";
+                       # first, drop any orphan value in child table
+                       if ($row->{onDelete} ne "RESTRICT") {
+                               my $sql = "delete from $table where $row->{key} not in (select $row->{foreignkey} from $row->{foreigntable})";
+                               $dbh->do($sql);
+                               print "SQL ERROR: $sql : $dbh->err \n" if $dbh->err;
+                       }
+                       my $sql="alter table $table ADD FOREIGN KEY $row->{key} ($row->{key}) REFERENCES $row->{foreigntable} ($row->{foreignkey})";
+                       $sql .= " on update ".$row->{onUpdate} if $row->{onUpdate};
+                       $sql .= " on delete ".$row->{onDelete} if $row->{onDelete};
+                       $dbh->do($sql);
+                       if ($dbh->err) {
+                               print "====================
+An error occured during :
+\t$sql
+It probably means there is something wrong in your DB : a row ($table.$row->{key}) refers to a value in $row->{foreigntable}.$row->{foreignkey} that does not exist. solve the problem and run updater again (or just the previous SQL statement).
+You can find those values with select
+\t$table.* from $table where $row->{key} not in (select $row->{foreignkey} from $row->{foreigntable})
+====================\n
+";
+                       }
+               }
+       }
 }
 
 #
@@ -530,7 +1384,7 @@ unless ($marcdone) {
        # changing marc field type
        $dbh->do('ALTER TABLE `biblioitems` CHANGE `marc` `marc` BLOB NULL DEFAULT NULL ');
        # adding marc xml, just for convenience
-       $dbh->do('ALTER TABLE `biblioitems` ADD `marcxml` TEXT NOT NULL');
+       $dbh->do('ALTER TABLE `biblioitems` ADD `marcxml` TEXT CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL ');
        # moving data from marc_subfield_value to biblio
        $sth = $dbh->prepare('select bibid,biblionumber from marc_biblio');
        $sth->execute;
@@ -538,13 +1392,17 @@ unless ($marcdone) {
        my $totaldone=0;
        while (my ($bibid,$biblionumber) = $sth->fetchrow) {
                my $record = MARCgetbiblio($dbh,$bibid);
-               $sth_update->execute($record->as_usmarc(),$record->as_xml(),$biblionumber);
+       #Force UTF-8 in record leader
+               $record->encoding('UTF-8');
+               print $record->as_formatted if ($biblionumber==3902);
+               $sth_update->execute($record->as_usmarc(),$record->as_xml_record(),$biblionumber);
                $totaldone++;
                print "\r$totaldone / $totaltodo" unless ($totaldone % 100);
        }
        print "\rdone\n";
 }
 
+
 # at last, remove useless fields
 foreach $table ( keys %uselessfields ) {
        my @fields = split /,/,$uselessfields{$table};
@@ -569,6 +1427,23 @@ foreach $table ( keys %uselessfields ) {
 }    # foreach
 
 
+# MOVE all tables TO UTF-8 and innoDB
+$sth = $dbh->prepare("show table status");
+$sth->execute;
+while ( my $table = $sth->fetchrow_hashref ) {
+#      if ($table->{Engine} ne 'InnoDB') {
+#              $dbh->do("ALTER TABLE $table->{Name} TYPE = innodb");
+#              print "moving $table->{Name} to InnoDB\n";
+#      }
+       unless ($table->{Collation} =~ /^utf8/) {
+               $dbh->do("ALTER TABLE $table->{Name} CONVERT TO CHARACTER SET utf8");
+               $dbh->do("ALTER TABLE $table->{Name} DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci");
+               # FIXME : maybe a ALTER TABLE tbl_name CONVERT TO CHARACTER SET utf8 would be better, def char set seems to work fine. If any problem encountered, let's try with convert !
+               print "moving $table->{Name} to utf8\n";
+       } else {
+       }
+}
+
 $sth->finish;
 
 #
@@ -578,6 +1453,7 @@ $sth->finish;
 # are still here, but uses other tables
 # (the ones that are filled by updatedatabase !)
 #
+
 sub MARCgetbiblio {
 
     # Returns MARC::Record of the biblio passed in parameter.
@@ -737,6 +1613,149 @@ sub MARCgetitem {
 exit;
 
 # $Log$
+# Revision 1.144  2006/06/08 15:36:31  alaurin
+# Add a new system preference 'AutomaticItemReturn' :
+#
+# if this prefence is switched on: the document returned in another library than homebranch, the system automaticly transfer the document to his homebranch (with notification for librarian in returns.tmpl) .
+#
+# switch off : the document stay in the holdingbranch ...
+#
+# correcting bugs :
+# - comment C4::acquisition (not using in request.pl).
+# - correcting date in request.pl
+# -add the new call of function getbranches in request.pl
+#
+# Revision 1.143  2006/06/07 02:02:47  bob_lyon
+# merging katipo changes...
+#
+# adding new preference IssuingInProcess
+#
+# Revision 1.142  2006/06/06 23:42:46  bob_lyon
+# Merging Katipo changes...
+#
+# Adding new system pref where one can still retrieve a correct reading
+# record history if one has moved older data from issues to oldissues table
+# to speed up issues speed
+#
+# Revision 1.141  2006/06/01 03:18:11  rangi
+# Adding a new column to the statistics table
+#
+# Revision 1.140  2006/05/22 22:40:45  rangi
+# Adding new systempreference allowing for the library to add borrowers to institutions (rest homes, parishes, schools, classes etc).
+#
+# Revision 1.139  2006/05/19 19:31:29  tgarip1957
+# Added new fields to auth_header and auth_subfield_table to allow ZEBRA use of authorities and new MARC framework like structure.
+# Authority tables are modified to be compatible with new MARC frameworks. This change is part of Authority Linking & Zebra authorities. Requires change in Mysql database. It will break head unless all changes regarding this is implemented. This warning will take place on all commits regarding this
+#
+# Revision 1.138  2006/05/19 16:51:44  alaurin
+# update database for :
+# - new feature ip and printer management
+# adding two fields in branches table (branchip,branchprinter)
+#
+# - waiting date : adding one field in reserves table(waiting date) to calculate the Maximum delay to pick up a reserved document when it's available
+#
+# new system preference :
+# - ReservesMaxPickUpDelay : Maximum delay to pick up a reserved document
+# TransfersMaxDaysWarning : Max delay before considering the transfer as potentialy a problem
+#
+# Revision 1.137  2006/04/18 09:36:36  plg
+# bug fixed: typo fixed in labels and labels_conf tables creation query.
+#
+# Revision 1.136  2006/04/17 21:55:33  sushi
+# Added 'labels' and 'labels_conf' tables, for spine lable tool.
+#
+# Revision 1.135  2006/04/15 02:37:03  tgarip1957
+# Marc record should be set to UTF-8 in leader.Force it.
+# XML should be with<record> wrappers
+#
+# Revision 1.134  2006/04/14 09:37:29  tipaul
+# improvements from SAN Ouest Provence :
+# * introducing a category_type into categories. It can be A (adult), C (children), P (Professionnal), I (institution/organisation).
+# * each category_type has it's own forms to create members.
+# * the borrowers table has been heavily modified (many fields changed), to get something more logic & readable
+# * reintroducing guarantor/guanrantee system that is now independant from hardcoded C/A for categories
+# * updating templates to fit template rules
+#
+# (see mail feb, 17 on koha-devel "new features for borrowers" for more details)
+#
+# Revision 1.133  2006/04/13 08:36:42  plg
+# new: function C4::Date::get_date_format_string_for_DHTMLcalendar based on
+# the system preference prefered date format.
+#
+# improvement: book fund list and budget list screen redesigned. Filters on
+# each field. Columns are not sortable yet. Using DHTML Calendar to fill date
+# fields instead of manual filling. Pagination system. From the book fund
+# list, you can reach the budget list, filtered on a book fund, or not. A
+# budget can be added only from book fund list screen.
+#
+# bug fixed: branchcode was missing in table aqbudget.
+#
+# bug fixed: when setting a branchcode to a book fund, all associated budgets
+# move to this branchcode.
+#
+# modification: when adding/modifying budget/fund, MySQL specific "REPLACE..."
+# statements replaced by standard SQL compliant statement.
+#
+# bug fixed: when adding/modifying a budget, if the book fund is associated to
+# a branch, the branch selection is disabled and set to the book fund branch.
+#
+# Revision 1.132  2006/04/06 12:37:05  hdl
+# Bugfixing : aqbookfund needed a field.
+#
+# Revision 1.131  2006/03/03 17:02:22  tipaul
+# commit for holidays and news management.
+# (some forgotten files)
+#
+# Revision 1.130  2006/03/03 16:35:21  tipaul
+# commit for holidays and news management.
+#
+# Contrib from Tümer Garip (from Turkey) :
+# * holiday :
+# in /tools/ the holiday.pl script let you define holidays (days where the library is closed), branch by branch. You can define 3 types of holidays :
+# - single day : only this day is closed
+# - repet weekly (like "sunday") : the day is holiday every week
+# - repet yearly (like "July, 4") : this day is closed every year.
+#
+# You can also put exception :
+# - sunday is holiday, but "2006 March, 5th" the library will be open
+#
+# The holidays are used for return date calculation : the return date is set to the next date where the library is open. A systempreference (useDaysMode) set ON (Calendar) or OFF (Normal) the calendar calculation.
+#
+# Revision 1.129  2006/02/27 18:19:33  hdl
+# New table used in overduerules.pl tools page.
+#
+# Revision 1.128  2006/01/25 15:16:06  tipaul
+# updating DB :
+# * removing useless tables
+# * adding useful indexes
+# * altering some columns definitions
+# * The goal being to have updater working fine for foreign keys.
+#
+# For me it's done, let me know if it works for you. You can see an updated schema of the DB (with constraints) on the wiki
+#
+# Revision 1.127  2006/01/24 17:57:17  tipaul
+# DB improvements : adding foreign keys on some tables. partial stuff done.
+#
+# Revision 1.126  2006/01/06 16:39:42  tipaul
+# synch'ing head and rel_2_2 (from 2.2.5, including npl templates)
+# Seems not to break too many things, but i'm probably wrong here.
+# at least, new features/bugfixes from 2.2.5 are here (tested on some features on my head local copy)
+#
+# - removing useless directories (koha-html and koha-plucene)
+#
+# Revision 1.125  2006/01/04 15:54:55  tipaul
+# utf8 is a : go for beta test in HEAD.
+# some explanations :
+# - updater/updatedatabase => will transform all tables in innoDB (not related to utf8, just to warn you) AND collate them in utf8 / utf8_general_ci. The SQL command is : ALTER TABLE tablename DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci.
+# - *-top.inc will show the pages in utf8
+# - THE HARD THING : for me, mysql-client and mysql-server were set up to communicate in iso8859-1, whatever the mysql collation ! Thus, pages were improperly shown, as datas were transmitted in iso8859-1 format ! After a full day of investigation, someone on usenet pointed "set NAMES 'utf8'" to explain that I wanted utf8. I could put this in my.cnf, but if I do that, ALL databases will "speak" in utf8, that's not what we want. Thus, I added a line in Context.pm : everytime a DB handle is opened, the communication is set to utf8.
+# - using marcxml field and no more the iso2709 raw marc biblioitems.marc field.
+#
+# Revision 1.124  2005/10/27 12:09:05  tipaul
+# new features for serial module :
+# - the last 5 issues are now shown, and their status can be changed (but not reverted to "waited", as there can be only one "waited")
+# - the library can create a "distribution list". this paper contains a list of borrowers (selected from the borrower list, or manually entered), and print it for a given issue. once printed, the sheet can be put on the issue and distributed to every reader on the list (one by one).
+#
 # Revision 1.123  2005/10/26 09:13:37  tipaul
 # big commit, still breaking things...
 #