Make stats page faster when file access is slow (#864)

* Fetch archive count from redis rather than looking at filesystem

* Store archive size in redis, use that in stats page

* Use MULTI for fetching archive sizes

* Avoid undefined warnings if DB lacks archive sizes

* Add redis connection as parameter to arcsize getters/setters

* Refresh archive size if file is modified

* Avoid crashing when uploading through webui

* Be more consistent when it comes to parameter order

lib/LANraragi/Model/Stats.pm

@@ -14,26 +14,8 @@ use LANraragi::Utils::Database qw(redis_decode redis_encode);
 use LANraragi::Utils::Logging qw(get_logger);
 
 sub get_archive_count {
-
-    #We can't trust the DB to contain the exact amount of files,
-    #As deleted files are still kept in store.
-    my $dirname = LANraragi::Model::Config->get_userdir;
-    my $count   = 0;
-
-    #Count files the old-fashioned way instead
-    find(
-        {   wanted => sub {
-                return if -d $_;    #Directories are excluded on the spot
-                if ( is_archive($_) ) {
-                    $count++;
-                }
-            },
-            no_chdir    => 1,
-            follow_fast => 1
-        },
-        $dirname
-    );
-    return $count;
+    my $redis  = LANraragi::Model::Config->get_redis_search;
+    return $redis->zcard("LRR_TITLES") + 0;    # Total number of archives (as int)
 }
 
 sub get_page_stat {
@@ -199,12 +181,23 @@ sub build_tag_stats {
 }
 
 sub compute_content_size {
+    my $redis_db = LANraragi::Model::Config->get_redis;
+
+    my @keys = $redis_db->keys('????????????????????????????????????????');
 
-    #Get size of archive folder
-    my $dirname = LANraragi::Model::Config->get_userdir;
-    my $size    = 0;
+    $redis_db->multi;
+    foreach my $id (@keys) {
+        LANraragi::Utils::Database::get_arcsize($redis_db, $id);
+    }
+    my @result = $redis_db->exec;
+    $redis_db->quit;
 
-    find( sub { $size += -s if -f }, $dirname );
+    my $size = 0;
+    foreach my $row (@result) {
+        if (defined($row)) {
+            $size = $size + $row;
+        }
+    }
 
     return int( $size / 1073741824 * 100 ) / 100;
 }

lib/LANraragi/Model/Upload.pm

@@ -123,6 +123,7 @@ sub handle_incoming_file {
     # (The file being physically present is necessary in case last modified time is used)
     LANraragi::Utils::Database::add_timestamp_tag( $redis, $id );
     LANraragi::Utils::Database::add_pagecount( $redis, $id );
+    LANraragi::Utils::Database::add_arcsize( $redis, $id );
     $redis->quit();
     $redis_search->quit();
 

lib/LANraragi/Utils/Database.pm

@@ -42,6 +42,9 @@ sub add_archive_to_redis ( $id, $file, $redis ) {
 
     $redis->hset( $id, "name", redis_encode($name) );
     $redis->hset( $id, "tags", "" );
+    if (defined($file) && -e $file) {
+        set_arcsize($redis, $id, -s $file);
+    }
 
     # Don't encode filenames.
     $redis->hset( $id, "file", $file );
@@ -68,6 +71,9 @@ sub change_archive_id ( $old_id, $new_id ) {
     if ( $redis->exists($old_id) ) {
         $redis->rename( $old_id, $new_id );
     }
+
+    my $file = $redis->hget($new_id, "file");
+    set_arcsize($redis, $new_id, -s $file);
     $redis->quit;
 
     # We also need to update categories that contain the ID.
@@ -590,4 +596,18 @@ sub get_tankoubons_by_file($arcid) {
     return @tankoubons;
 }
 
+sub add_arcsize ( $redis, $id ) {
+    my $file = $redis->hget( $id, "file" );
+    my $arcsize = -s $file;
+    set_arcsize( $redis, $id, $arcsize );
+}
+
+sub set_arcsize( $redis, $id, $arcsize ) {
+    $redis->hset( $id, "arcsize", $arcsize );
+}
+
+sub get_arcsize ( $redis, $id ) {
+    return $redis->hget( $id, "arcsize" );
+}
+
 1;

lib/Shinobu.pm

@@ -260,6 +260,11 @@ sub add_to_filemap ( $redis_cfg, $file ) {
                 invalidate_cache();
             }
 
+            unless ( LANraragi::Utils::Database::get_arcsize( $redis_arc, $id ) ) {
+                $logger->debug("arcsize is not set for $id, storing now!");
+                LANraragi::Utils::Database::add_arcsize( $redis_arc, $id );
+            }
+
             # Set pagecount in case it's not already there
             unless ( $redis_arc->hget( $id, "pagecount" ) ) {
                 $logger->debug("Pagecount not calculated for $id, doing it now!");