Unverified Commit b9f2e1a9 authored by IKEDA Soji's avatar IKEDA Soji Committed by GitHub
Browse files

Merge pull request #388 from ikedas/issue-368 by ikedas

Rename a list takes incredible time #368
parents bd057ed6 4e38c715
......@@ -45,7 +45,8 @@ use Sympa::Tools::File;
my $log = Sympa::Log->instance;
sub new {
my $class = shift;
my $class = shift;
my %options = @_;
my $self = bless {
msg_directory => $Conf::Conf{'queuebulk'} . '/msg',
......@@ -58,6 +59,14 @@ sub new {
$self->_create_spool;
# Build glob pattern (for pct entries).
$self->{_glob_pattern} = Sympa::Spool::build_glob_pattern(
'%s.%s.%d.%f.%s@%s_%s,%ld,%d/%s',
[ qw(priority packet_priority date time localpart domainpart tag pid rand serial)
],
%options
) || '*/*';
return $self;
}
......@@ -102,7 +111,7 @@ sub next {
!/,lock/
and !m{(?:\A|/)(?:\.|T\.|BAD-)}
and -f ($self->{pct_directory} . '/' . $_)
} glob '*/*'
} glob $self->{_glob_pattern}
];
chdir $cwd;
}
......
......@@ -44,6 +44,7 @@ use Sympa::Spool::Digest::Collection;
use Sympa::Spool::Held;
use Sympa::Spool::Incoming;
use Sympa::Spool::Moderation;
use Sympa::Task;
use Sympa::Tools::File;
use base qw(Sympa::Request::Handler);
......@@ -350,13 +351,12 @@ sub _move {
# Continue even if there are some troubles.
#FIXME: Refactor to use Sympa::Spool subclass.
$queue = $Conf::Conf{'queuetask'};
unless (opendir $dh, $queue) {
$log->syslog('err', 'Unable to open task spool %s: %m', $queue);
} else {
if (Sympa::Task::list_tasks($queue, $current_list->get_id)) {
my $current_list_id = $current_list->get_id;
my $new_list_id = $fake_list->get_id;
foreach my $file (sort readdir $dh) {
foreach my $task (Sympa::Task::get_tasks_by_list($current_list_id)) {
my $file = $task->{'filename'};
next
unless $file =~
/^(\d+)\.(\w*)\.(\w+)\.([^\s\@]+)(?:\@([\w\.\-]+))?$/;
......@@ -373,8 +373,6 @@ sub _move {
$queue, $file, $newfile);
}
}
close $dh;
}
# Rename files in topic spool.
......@@ -404,7 +402,7 @@ sub _move {
# Rename files in outgoing spool.
# Continue even if there are some troubles.
my $spool = Sympa::Bulk->new;
my $spool = Sympa::Bulk->new(context => $current_list);
while (1) {
my ($message, $handle) = $spool->next(no_filter => 1);
last unless $handle;
......
......@@ -35,6 +35,8 @@ use base qw(Sympa::Spool);
my $log = Sympa::Log->instance;
use constant _no_glob_pattern => 1; # Not a filesystem spool.
sub next {
my $self = shift;
......
......@@ -57,12 +57,22 @@ sub new {
my $self = bless {
%options,
%{$class->_directories(%options) || {}},
_metadatas => undef,
_metadatas => undef,
_glob_pattern => undef,
} => $class;
$self->_create;
$self->_init(0) or return undef;
# Build glob pattern (using encoded attributes).
unless ($self->_no_glob_pattern) {
my $opts = {%options};
$self->_filter_pre($opts);
$self->{_glob_pattern} =
Sympa::Spool::build_glob_pattern($self->_marshal_format,
$self->_marshal_keys, %$opts);
}
$self;
}
......@@ -93,6 +103,8 @@ sub _create {
sub _init {1}
sub _no_glob_pattern {0}
sub marshal {
my $self = shift;
my $message = shift;
......@@ -167,11 +179,11 @@ sub _load {
my $self = shift;
my @entries;
if ($self->_glob_pattern) {
if ($self->{_glob_pattern}) {
my $cwd = Cwd::getcwd();
die sprintf 'Cannot chdir to %s: %s', $self->{directory}, $ERRNO
unless chdir $self->{directory};
@entries = glob $self->_glob_pattern;
@entries = glob $self->{_glob_pattern};
chdir $cwd;
} else {
my $dh;
......@@ -194,8 +206,6 @@ sub _load {
return $metadatas;
}
sub _glob_pattern {undef}
sub _is_collection {0}
sub quarantine {
......@@ -884,10 +894,7 @@ generator class must implement dup(), new() and to_string().
=item _glob_pattern ( )
I<Instance method>.
If implemented and returns non-empty string,
glob() is used to search entries in the spool.
Otherwise readdir() is used for filesystem spool to get all entries.
Deprecated. See _no_glob_pattern ( )
=item _init ( $state )
......@@ -922,6 +929,14 @@ _marshal_format() and _marshal_keys() are used to marshal metadata.
_marshal_keys() and _marshal_regexp() are used to unmarshal metadata.
See also marshal_metadata() and unmarshal_metadata().
=item _no_glob_pattern ( )
I<Class or instance method>, I<overridable for filesystem spool>.
If it returns false value,
glob() is used as much as possible to scan the spool faster.
Otherwise readdir() is used for filesystem spool to get all entries.
By default returns false value.
=item _store_key ( )
I<Instance method>.
......@@ -1012,5 +1027,7 @@ were introduced on Sympa 6.2.8.
_filter_pre() was introduced on Sympa 6.2.10.
marshal(), unmarshal() and C<no_filter> option of next()
were introduced on Sympa 6.2.22.
_no_glob_pattern() was introduced and _glob_pattern() was deprecated
on Sympa 6.2.36.
=cut
......@@ -65,8 +65,6 @@ sub _filter_pre {
use constant _generator => 'Sympa::Request';
sub _glob_pattern { shift->{_pattern} }
use constant _marshal_format => '%ld,%s@%s_%s,%s,%s';
use constant _marshal_keys =>
[qw(date localpart domainpart KEYAUTH email action)];
......@@ -74,24 +72,6 @@ use constant _marshal_regexp =>
qr{\A(\d+),([^\s\@]+)\@([-.\w]+)_([\da-f]+),([^\s,]*),(\w+)\z};
use constant _store_key => 'keyauth';
sub new {
my $class = shift;
my %options = @_;
my $self = $class->SUPER::new(%options);
# Build glob pattern using encoded e-mail.
if ($self) {
my $opts = {%options};
$self->_filter_pre($opts);
$self->{_pattern} =
Sympa::Spool::build_glob_pattern($self->_marshal_format,
$self->_marshal_keys, %$opts);
}
$self;
}
1;
__END__
......
......@@ -80,6 +80,8 @@ use constant _marshal_format => '%ld.%f,%ld,%d';
use constant _marshal_keys => [qw(date TIME PID RAND)];
use constant _marshal_regexp => qr{\A(\d+)\.(\d+\.\d+)(?:,.*)?\z};
use constant _no_glob_pattern => 1;
sub next {
my $self = shift;
......
......@@ -37,25 +37,11 @@ sub _directories {
use constant _generator => 'Sympa::Message';
sub _glob_pattern { shift->{_pattern} }
use constant _marshal_format => '%s@%s_%s';
use constant _marshal_keys => [qw(localpart domainpart AUTHKEY)];
use constant _marshal_regexp => qr{\A([^\s\@]+)\@([-.\w]+)_([\da-f]+)\z};
use constant _store_key => 'authkey';
sub new {
my $class = shift;
my %options = @_;
my $self = $class->SUPER::new(%options);
$self->{_pattern} =
Sympa::Spool::build_glob_pattern($self->_marshal_format,
$self->_marshal_keys, %options);
$self;
}
1;
__END__
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment