Merge pull request #388 from ikedas/issue-368 by ikedas

Rename a list takes incredible time #368

src/lib/Sympa/Bulk.pm

@@ -45,7 +45,8 @@ use Sympa::Tools::File;
 my $log = Sympa::Log->instance;
 
 sub new {
-    my $class = shift;
+    my $class   = shift;
+    my %options = @_;
 
     my $self = bless {
         msg_directory     => $Conf::Conf{'queuebulk'} . '/msg',
@@ -58,6 +59,14 @@ sub new {
 
     $self->_create_spool;
 
+    # Build glob pattern (for pct entries).
+    $self->{_glob_pattern} = Sympa::Spool::build_glob_pattern(
+        '%s.%s.%d.%f.%s@%s_%s,%ld,%d/%s',
+        [   qw(priority packet_priority date time localpart domainpart tag pid rand serial)
+        ],
+        %options
+    ) || '*/*';
+
     return $self;
 }
 
@@ -102,7 +111,7 @@ sub next {
                         !/,lock/
                     and !m{(?:\A|/)(?:\.|T\.|BAD-)}
                     and -f ($self->{pct_directory} . '/' . $_)
-            } glob '*/*'
+            } glob $self->{_glob_pattern}
         ];
         chdir $cwd;
     }

src/lib/Sympa/Request/Handler/move_list.pm

@@ -44,6 +44,7 @@ use Sympa::Spool::Digest::Collection;
 use Sympa::Spool::Held;
 use Sympa::Spool::Incoming;
 use Sympa::Spool::Moderation;
+use Sympa::Task;
 use Sympa::Tools::File;
 
 use base qw(Sympa::Request::Handler);
@@ -350,13 +351,12 @@ sub _move {
     # Continue even if there are some troubles.
     #FIXME: Refactor to use Sympa::Spool subclass.
     $queue = $Conf::Conf{'queuetask'};
-    unless (opendir $dh, $queue) {
-        $log->syslog('err', 'Unable to open task spool %s: %m', $queue);
-    } else {
+    if (Sympa::Task::list_tasks($queue, $current_list->get_id)) {
         my $current_list_id = $current_list->get_id;
         my $new_list_id     = $fake_list->get_id;
 
-        foreach my $file (sort readdir $dh) {
+        foreach my $task (Sympa::Task::get_tasks_by_list($current_list_id)) {
+            my $file = $task->{'filename'};
             next
                 unless $file =~
                 /^(\d+)\.(\w*)\.(\w+)\.([^\s\@]+)(?:\@([\w\.\-]+))?$/;
@@ -373,8 +373,6 @@ sub _move {
                     $queue, $file, $newfile);
             }
         }
-
-        close $dh;
     }
 
     # Rename files in topic spool.
@@ -404,7 +402,7 @@ sub _move {
 
     # Rename files in outgoing spool.
     # Continue even if there are some troubles.
-    my $spool = Sympa::Bulk->new;
+    my $spool = Sympa::Bulk->new(context => $current_list);
     while (1) {
         my ($message, $handle) = $spool->next(no_filter => 1);
         last unless $handle;

src/lib/Sympa/Request/Message.pm

@@ -35,6 +35,8 @@ use base qw(Sympa::Spool);
 
 my $log = Sympa::Log->instance;
 
+use constant _no_glob_pattern => 1;     # Not a filesystem spool.
+
 sub next {
     my $self = shift;
 

src/lib/Sympa/Spool.pm

@@ -57,12 +57,22 @@ sub new {
     my $self = bless {
         %options,
         %{$class->_directories(%options) || {}},
-        _metadatas => undef,
+        _metadatas    => undef,
+        _glob_pattern => undef,
     } => $class;
 
     $self->_create;
     $self->_init(0) or return undef;
 
+    # Build glob pattern (using encoded attributes).
+    unless ($self->_no_glob_pattern) {
+        my $opts = {%options};
+        $self->_filter_pre($opts);
+        $self->{_glob_pattern} =
+            Sympa::Spool::build_glob_pattern($self->_marshal_format,
+            $self->_marshal_keys, %$opts);
+    }
+
     $self;
 }
 
@@ -93,6 +103,8 @@ sub _create {
 
 sub _init {1}
 
+sub _no_glob_pattern {0}
+
 sub marshal {
     my $self    = shift;
     my $message = shift;
@@ -167,11 +179,11 @@ sub _load {
     my $self = shift;
 
     my @entries;
-    if ($self->_glob_pattern) {
+    if ($self->{_glob_pattern}) {
         my $cwd = Cwd::getcwd();
         die sprintf 'Cannot chdir to %s: %s', $self->{directory}, $ERRNO
             unless chdir $self->{directory};
-        @entries = glob $self->_glob_pattern;
+        @entries = glob $self->{_glob_pattern};
         chdir $cwd;
     } else {
         my $dh;
@@ -194,8 +206,6 @@ sub _load {
     return $metadatas;
 }
 
-sub _glob_pattern {undef}
-
 sub _is_collection {0}
 
 sub quarantine {
@@ -884,10 +894,7 @@ generator class must implement dup(), new() and to_string().
 
 =item _glob_pattern ( )
 
-I<Instance method>.
-If implemented and returns non-empty string,
-glob() is used to search entries in the spool.
-Otherwise readdir() is used for filesystem spool to get all entries.
+Deprecated.  See _no_glob_pattern ( )
 
 =item _init ( $state )
 
@@ -922,6 +929,14 @@ _marshal_format() and _marshal_keys() are used to marshal metadata.
 _marshal_keys() and _marshal_regexp() are used to unmarshal metadata.
 See also marshal_metadata() and unmarshal_metadata().
 
+=item _no_glob_pattern ( )
+
+I<Class or instance method>, I<overridable for filesystem spool>.
+If it returns false value,
+glob() is used as much as possible to scan the spool faster.
+Otherwise readdir() is used for filesystem spool to get all entries.
+By default returns false value.
+
 =item _store_key ( )
 
 I<Instance method>.
@@ -1012,5 +1027,7 @@ were introduced on Sympa 6.2.8.
 _filter_pre() was introduced on Sympa 6.2.10.
 marshal(), unmarshal() and C<no_filter> option of next()
 were introduced on Sympa 6.2.22.
+_no_glob_pattern() was introduced and _glob_pattern() was deprecated
+on Sympa 6.2.36.
 
 =cut

src/lib/Sympa/Spool/Auth.pm

@@ -65,8 +65,6 @@ sub _filter_pre {
 
 use constant _generator => 'Sympa::Request';
 
-sub _glob_pattern { shift->{_pattern} }
-
 use constant _marshal_format => '%ld,%s@%s_%s,%s,%s';
 use constant _marshal_keys =>
     [qw(date localpart domainpart KEYAUTH email action)];
@@ -74,24 +72,6 @@ use constant _marshal_regexp =>
     qr{\A(\d+),([^\s\@]+)\@([-.\w]+)_([\da-f]+),([^\s,]*),(\w+)\z};
 use constant _store_key => 'keyauth';
 
-sub new {
-    my $class   = shift;
-    my %options = @_;
-
-    my $self = $class->SUPER::new(%options);
-
-    # Build glob pattern using encoded e-mail.
-    if ($self) {
-        my $opts = {%options};
-        $self->_filter_pre($opts);
-        $self->{_pattern} =
-            Sympa::Spool::build_glob_pattern($self->_marshal_format,
-            $self->_marshal_keys, %$opts);
-    }
-
-    $self;
-}
-
 1;
 __END__
 

src/lib/Sympa/Spool/Digest.pm

@@ -80,6 +80,8 @@ use constant _marshal_format => '%ld.%f,%ld,%d';
 use constant _marshal_keys   => [qw(date TIME PID RAND)];
 use constant _marshal_regexp => qr{\A(\d+)\.(\d+\.\d+)(?:,.*)?\z};
 
+use constant _no_glob_pattern => 1;
+
 sub next {
     my $self = shift;
 

src/lib/Sympa/Spool/Held.pm

@@ -37,25 +37,11 @@ sub _directories {
 
 use constant _generator => 'Sympa::Message';
 
-sub _glob_pattern { shift->{_pattern} }
-
 use constant _marshal_format => '%s@%s_%s';
 use constant _marshal_keys   => [qw(localpart domainpart AUTHKEY)];
 use constant _marshal_regexp => qr{\A([^\s\@]+)\@([-.\w]+)_([\da-f]+)\z};
 use constant _store_key      => 'authkey';
 
-sub new {
-    my $class   = shift;
-    my %options = @_;
-
-    my $self = $class->SUPER::new(%options);
-    $self->{_pattern} =
-        Sympa::Spool::build_glob_pattern($self->_marshal_format,
-        $self->_marshal_keys, %options);
-
-    $self;
-}
-
 1;
 __END__