Fix bugs, add transfer rate monitor, add graceful stop, support single-use URLs in rippers

build.gradle.kts

@@ -5,13 +5,14 @@
 // gradle clean build -PjavacRelease=21
 // gradle clean build -PcustomVersion=1.0.0-10-asdf
 val customVersion = (project.findProperty("customVersion") ?: "") as String
-val javacRelease = (project.findProperty("javacRelease") ?: "17") as String
+val javacRelease = (project.findProperty("javacRelease") ?: "21") as String
 
 plugins {
   id("fr.brouillard.oss.gradle.jgitver") version "0.9.1"
   id("jacoco")
   id("java")
   id("maven-publish")
+  id("com.gradleup.shadow") version "9.0.0-rc1"
 }
 
 repositories {
@@ -23,11 +24,11 @@ dependencies {
   implementation("com.lmax:disruptor:3.4.4")
   implementation("org.java-websocket:Java-WebSocket:1.5.3")
   implementation("org.jsoup:jsoup:1.16.1")
-  implementation("org.json:json:20211205")
+  implementation("org.json:json:20231013")
   implementation("com.j2html:j2html:1.6.0")
   implementation("commons-configuration:commons-configuration:1.10")
   implementation("commons-cli:commons-cli:1.5.0")
-  implementation("commons-io:commons-io:2.13.0")
+  implementation("commons-io:commons-io:2.14.0")
   implementation("org.apache.httpcomponents:httpclient:4.5.14")
   implementation("org.apache.httpcomponents:httpmime:4.5.14")
   implementation("org.apache.logging.log4j:log4j-api:2.20.0")
@@ -36,13 +37,19 @@ dependencies {
   implementation("org.graalvm.js:js:22.3.2")
   testImplementation(enforcedPlatform("org.junit:junit-bom:5.10.0"))
   testImplementation("org.junit.jupiter:junit-jupiter")
+  testImplementation("org.mockito:mockito-core:5.+")
   testRuntimeOnly("org.junit.platform:junit-platform-launcher")
 }
 
 group = "com.rarchives.ripme"
 version = "1.7.94"
 description = "ripme"
 
+java {
+  sourceCompatibility = JavaVersion.VERSION_21
+  targetCompatibility = JavaVersion.VERSION_21
+}
+
 jacoco {
   toolVersion = "0.8.12"
 }
@@ -80,6 +87,10 @@ tasks.withType<Jar> {
   })
 }
 
+tasks.shadowJar {
+  transform<com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer>()
+}
+
 publishing {
   publications {
     create<MavenPublication>("maven") {
@@ -105,6 +116,8 @@ tasks.test {
     includeEngines("junit-vintage")
   }
   finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run
+
+  jvmArgs("-javaagent:${classpath.find { it.name.contains("mockito") }?.absolutePath}")
 }
 
 tasks.register<Test>("testAll") {

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

src/main/java/com/rarchives/ripme/App.java

@@ -76,6 +76,9 @@ public static void main(String[] args) throws IOException {
         if (GraphicsEnvironment.isHeadless() || args.length > 0) {
             handleArguments(args);
         } else {
+            // Antialiasing hint, especially for Linux
+            System.setProperty("awt.useSystemAAFontSettings", "on");
+
             if (SystemUtils.IS_OS_MAC_OSX) {
                 System.setProperty("apple.laf.useScreenMenuBar", "true");
                 System.setProperty("com.apple.mrj.application.apple.menu.about.name", "RipMe");

src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java

@@ -13,11 +13,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -36,9 +32,6 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
 
     private static final Logger logger = LogManager.getLogger(AbstractHTMLRipper.class);
 
-    private final Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<>());
-    private final Map<URL, Path> itemsCompleted = Collections.synchronizedMap(new HashMap<>());
-    private final Map<URL, String> itemsErrored = Collections.synchronizedMap(new HashMap<>());
     Document cachedFirstPage;
 
     protected AbstractHTMLRipper(URL url) throws IOException {
@@ -76,10 +69,6 @@ protected List<String> getDescriptionsFromPage(Document doc) throws IOException
 
     protected abstract void downloadURL(URL url, int index);
 
-    protected DownloadThreadPool getThreadPool() {
-        return null;
-    }
-
     protected boolean keepSortOrder() {
         return true;
     }
@@ -121,7 +110,7 @@ protected boolean pageContainsAlbums(URL url) {
 
     @Override
     public void rip() throws IOException, URISyntaxException {
-        int index = 0;
+        int imageIndex = 0;
         int textindex = 0;
         logger.info("Retrieving " + this.url);
         sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
@@ -176,9 +165,10 @@ public void rip() throws IOException, URISyntaxException {
                 }
 
                 for (String imageURL : imageURLs) {
-                    index += 1;
-                    logger.debug("Found image url #" + index + ": '" + imageURL + "'");
-                    downloadURL(new URI(imageURL).toURL(), index);
+                    imageIndex += 1;
+                    logger.debug("Found image url #{} of album {}: {}", imageIndex, this.url, imageURL);
+                    setItemsTotal(Math.max(getItemsTotal(), imageIndex));
+                    downloadURL(new URI(imageURL).toURL(), imageIndex);
                     if (isStopped() || isThisATest()) {
                         break;
                     }
@@ -206,7 +196,7 @@ public void rip() throws IOException, URISyntaxException {
                                 workingDir.getCanonicalPath()
                                         + ""
                                         + File.separator
-                                        + getPrefix(index)
+                                        + getPrefix(imageIndex)
                                         + (tempDesc.length > 1 ? tempDesc[1] : filename)
                                         + ".txt").exists();
 
@@ -236,12 +226,17 @@ public void rip() throws IOException, URISyntaxException {
             }
         }
 
-        // If they're using a thread pool, wait for it.
-        if (getThreadPool() != null) {
-            logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName());
-            getThreadPool().waitForThreads();
+        logger.info("All items queued; total items: {}; url: {}", imageIndex, url);
+
+        // Final total item count is now known
+        setItemsTotal(imageIndex);
+
+        if (getCrawlerThreadPool() != null) {
+            logger.debug("Waiting for crawler threadpool: {}", url);
+            getCrawlerThreadPool().waitForThreads(imageIndex, shouldStop, url);
         }
-        waitForThreads();
+
+        waitForRipperThreads();
     }
 
     /**
@@ -338,68 +333,6 @@ protected boolean allowDuplicates() {
         return false;
     }
 
-    @Override
-    /*
-      Returns total amount of files attempted.
-     */
-    public int getCount() {
-        return itemsCompleted.size() + itemsErrored.size();
-    }
-
-    @Override
-    /*
-      Queues multiple URLs of single images to download from a single Album URL
-     */
-    public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
-        // Only download one file if this is a test.
-        if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
-            stop();
-            itemsPending.clear();
-            return false;
-        }
-        if (!allowDuplicates()
-                && ( itemsPending.containsKey(url)
-                  || itemsCompleted.containsKey(url)
-                  || itemsErrored.containsKey(url) )) {
-            // Item is already downloaded/downloading, skip it.
-            logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
-            return false;
-        }
-        if (shouldIgnoreURL(url)) {
-            sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension");
-            return false;
-        }
-        if (Utils.getConfigBoolean("urls_only.save", false)) {
-            // Output URL to file
-            Path urlFile = Paths.get(this.workingDir + "/urls.txt");
-            String text = url.toExternalForm() + System.lineSeparator();
-            try {
-                Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND);
-                itemsCompleted.put(url, urlFile);
-            } catch (IOException e) {
-                logger.error("Error while writing to " + urlFile, e);
-            }
-        }
-        else {
-            itemsPending.put(url, saveAs.toFile());
-            DownloadFileThread dft = new DownloadFileThread(url,  saveAs.toFile(),  this, getFileExtFromMIME);
-            if (referrer != null) {
-                dft.setReferrer(referrer);
-            }
-            if (cookies != null) {
-                dft.setCookies(cookies);
-            }
-            threadPool.addThread(dft);
-        }
-
-        return true;
-    }
-
-    @Override
-    public boolean addURLToDownload(URL url, Path saveAs) {
-        return addURLToDownload(url, saveAs, null, null, false);
-    }
-
     /**
      * Queues image to be downloaded and saved.
      * Uses filename from URL to decide filename.
@@ -413,72 +346,6 @@ protected boolean addURLToDownload(URL url) {
         return addURLToDownload(url, "", "");
     }
 
-    @Override
-    /*
-      Cleans up & tells user about successful download
-     */
-    public void downloadCompleted(URL url, Path saveAs) {
-        if (observer == null) {
-            return;
-        }
-        try {
-            String path = Utils.removeCWD(saveAs);
-            RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
-            itemsPending.remove(url);
-            itemsCompleted.put(url, saveAs);
-            observer.update(this, msg);
-
-            checkIfComplete();
-        } catch (Exception e) {
-            logger.error("Exception while updating observer: ", e);
-        }
-    }
-
-    @Override
-    /*
-     * Cleans up & tells user about failed download.
-     */
-    public void downloadErrored(URL url, String reason) {
-        if (observer == null) {
-            return;
-        }
-        itemsPending.remove(url);
-        itemsErrored.put(url, reason);
-        observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
-
-        checkIfComplete();
-    }
-
-    @Override
-    /*
-      Tells user that a single file in the album they wish to download has
-      already been downloaded in the past.
-     */
-    public void downloadExists(URL url, Path file) {
-        if (observer == null) {
-            return;
-        }
-
-        itemsPending.remove(url);
-        itemsCompleted.put(url, file);
-        observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file));
-
-        checkIfComplete();
-    }
-
-    /**
-     * Notifies observers and updates state if all files have been ripped.
-     */
-    @Override
-    protected void checkIfComplete() {
-        if (observer == null) {
-            return;
-        }
-        if (itemsPending.isEmpty()) {
-            super.checkIfComplete();
-        }
-    }
-
     /**
      * Sets directory to save all ripped files to.
      * @param url
@@ -515,22 +382,20 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException {
      */
     @Override
     public int getCompletionPercentage() {
-        double total = itemsPending.size()  + itemsErrored.size() + itemsCompleted.size();
-        return (int) (100 * ( (total - itemsPending.size()) / total));
+        double total = getTotalCount();
+        if (total == 0) {
+            return 0;
+        }
+        return (int) (100 * ( (itemsCompleted.size() + itemsErrored.size()) / total));
     }
 
-    /**
-     * @return
-     *      Human-readable information on the status of the current rip.
-     */
     @Override
-    public String getStatusText() {
-        return getCompletionPercentage() +
-                "% " +
-                "- Pending: " + itemsPending.size() +
-                ", Completed: " + itemsCompleted.size() +
-                ", Errored: " + itemsErrored.size();
+    public int getPendingCount() {
+        DownloadThreadPool threadPool = getRipperThreadPool();
+        if (threadPool != null) {
+            return threadPool.getPendingThreadCount();
+        }
+        return itemsPending.size();
     }
 
-
 }