speed boost: tools stage and commit bib records

Turned off autocommit; commit every 50 records.

Signed-off-by: Chris Cormack <crc@liblime.com>
Signed-off-by: Joshua Ferraro <jmf@liblime.com>

diff --git a/tools/manage-marc-import.pl b/tools/manage-marc-import.pl
index ebc6b5c..3a7bd4e 100755 (executable)
--- a/tools/manage-marc-import.pl
+++ b/tools/manage-marc-import.pl
@@ -152,13 +152,15 @@ sub commit_batch {
     my ($template, $import_batch_id) = @_;
 
     my $job = undef;
     my ($template, $import_batch_id) = @_;
 
     my $job = undef;

+    $dbh->{AutoCommit} = 0;

     my $callback = sub {};
     if ($runinbackground) {
         $job = put_in_background($import_batch_id);
     my $callback = sub {};
     if ($runinbackground) {
         $job = put_in_background($import_batch_id);

-        $callback = progress_callback($job);
+        $callback = progress_callback($job, $dbh);

     }
     my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = 
         BatchCommitBibRecords($import_batch_id, 50, $callback);
     }
     my ($num_added, $num_updated, $num_items_added, $num_items_errored, $num_ignored) = 
         BatchCommitBibRecords($import_batch_id, 50, $callback);

+    $dbh->commit();

 
     my $results = {
         did_commit => 1,
 
     my $results = {
         did_commit => 1,


@@ -178,14 +180,16 @@ sub commit_batch {
 sub revert_batch {
     my ($template, $import_batch_id) = @_;
 
 sub revert_batch {
     my ($template, $import_batch_id) = @_;
 

+    $dbh->{AutoCommit} = 0;

     my $job = undef;
     my $callback = sub {};
     if ($runinbackground) {
         $job = put_in_background($import_batch_id);
     my $job = undef;
     my $callback = sub {};
     if ($runinbackground) {
         $job = put_in_background($import_batch_id);

-        $callback = progress_callback($job);
+        $callback = progress_callback($job, $dbh);

     }
     my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) = 
         BatchRevertBibRecords($import_batch_id, 50, $callback);
     }
     my ($num_deleted, $num_errors, $num_reverted, $num_items_deleted, $num_ignored) = 
         BatchRevertBibRecords($import_batch_id, 50, $callback);

+    $dbh->commit();

 
     my $results = {
         did_revert => 1,
 
     my $results = {
         did_revert => 1,


@@ -239,9 +243,11 @@ sub put_in_background {
 
 sub progress_callback {
     my $job = shift;
 
 sub progress_callback {
     my $job = shift;

+    my $dbh = shift;

     return sub {
         my $progress = shift;
         $job->progress($progress);
     return sub {
         my $progress = shift;
         $job->progress($progress);

+        $dbh->commit();

     }
 }
 
     }
 }
 

diff --git a/tools/stage-marc-import.pl b/tools/stage-marc-import.pl
index 16ba5fa..e75019a 100755 (executable)
--- a/tools/stage-marc-import.pl
+++ b/tools/stage-marc-import.pl
@@ -44,6 +44,7 @@ use C4::BackgroundJob;
 
 my $input = new CGI;
 my $dbh = C4::Context->dbh;
 
 my $input = new CGI;
 my $dbh = C4::Context->dbh;

+$dbh->{AutoCommit} = 0;

 
 my $fileID=$input->param('uploadedfileid');
 my $runinbackground = $input->param('runinbackground');
 
 my $fileID=$input->param('uploadedfileid');
 my $runinbackground = $input->param('runinbackground');


@@ -117,15 +118,16 @@ if ($completedJobID) {
 
         # if we get here, we're a child that has detached
         # itself from Apache
 
         # if we get here, we're a child that has detached
         # itself from Apache

-        $staging_callback = staging_progress_callback($job);
-        $matching_callback = matching_progress_callback($job);
+        $staging_callback = staging_progress_callback($job, $dbh);
+        $matching_callback = matching_progress_callback($job, $dbh);

 
     }
 
     # FIXME branch code
     my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename, 
                                                                                    $comments, '', $parse_items, 0,
 
     }
 
     # FIXME branch code
     my ($batch_id, $num_valid, $num_items, @import_errors) = BatchStageMarcRecords($syntax, $marcrecord, $filename, 
                                                                                    $comments, '', $parse_items, 0,

-                                                                                   50, staging_progress_callback($job));
+                                                                                   50, staging_progress_callback($job, $dbh));
+    $dbh->commit();

     my $num_with_matches = 0;
     my $checked_matches = 0;
     my $matcher_failed = 0;
     my $num_with_matches = 0;
     my $checked_matches = 0;
     my $matcher_failed = 0;


@@ -135,8 +137,9 @@ if ($completedJobID) {
         if (defined $matcher) {
             $checked_matches = 1;
             $matcher_code = $matcher->code();
         if (defined $matcher) {
             $checked_matches = 1;
             $matcher_code = $matcher->code();

-            $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job));
+            $num_with_matches = BatchFindBibDuplicates($batch_id, $matcher, 10, 50, matching_progress_callback($job, $dbh));

             SetImportBatchMatcher($batch_id, $matcher_id);
             SetImportBatchMatcher($batch_id, $matcher_id);

+            $dbh->commit();

         } else {
             $matcher_failed = 1;
         }
         } else {
             $matcher_failed = 1;
         }


@@ -180,17 +183,21 @@ exit 0;
 
 sub staging_progress_callback {
     my $job = shift;
 
 sub staging_progress_callback {
     my $job = shift;

+    my $dbh = shift;

     return sub {
         my $progress = shift;
         $job->progress($progress);
     return sub {
         my $progress = shift;
         $job->progress($progress);

+        $dbh->commit();

     }
 }
 
 sub matching_progress_callback {
     my $job = shift;
     }
 }
 
 sub matching_progress_callback {
     my $job = shift;

+    my $dbh = shift;

     my $start_progress = $job->progress();
     return sub {
         my $progress = shift;
         $job->progress($start_progress + $progress);
     my $start_progress = $job->progress();
     return sub {
         my $progress = shift;
         $job->progress($start_progress + $progress);

+        $dbh->commit();

     }
 }
     }
 }