From c186821942e687a4e70e6085c372f5fe67824a49 Mon Sep 17 00:00:00 2001 From: JFronny <33260128+JFronny@users.noreply.github.com> Date: Fri, 7 May 2021 17:39:13 +0200 Subject: [PATCH] Multi-thread download tasks --- gradle.properties | 6 +-- .../io/gitlab/jfronny/resclone/Resclone.java | 43 +++++++++++++------ .../resclone/fetchers/CurseforgeFetcher.java | 3 +- .../processors/PruneVanillaProcessor.java | 2 +- .../processors/RemoveEmptyProcessor.java | 2 +- .../processors/RootPathProcessor.java | 2 +- .../resclone/{ => util}/PackUrlCache.java | 2 +- .../{ => util}/io/MoveDirVisitor.java | 2 +- .../{ => util}/io/PathPruneVisitor.java | 2 +- .../{ => util}/io/RemoveDirVisitor.java | 2 +- 10 files changed, 41 insertions(+), 25 deletions(-) rename src/main/java/io/gitlab/jfronny/resclone/{ => util}/PackUrlCache.java (96%) rename src/main/java/io/gitlab/jfronny/resclone/{ => util}/io/MoveDirVisitor.java (96%) rename src/main/java/io/gitlab/jfronny/resclone/{ => util}/io/PathPruneVisitor.java (95%) rename src/main/java/io/gitlab/jfronny/resclone/{ => util}/io/RemoveDirVisitor.java (93%) diff --git a/gradle.properties b/gradle.properties index a0a1504..fb26a43 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,12 +3,12 @@ org.gradle.jvmargs=-Xmx1G # Fabric Properties # check these on https://modmuss50.me/fabric.html minecraft_version=1.16.5 -yarn_mappings=1.16.5+build.6 +yarn_mappings=1.16.5+build.8 loader_version=0.11.3 # Mod Properties -mod_version=1.5.0 +mod_version=1.6.0 maven_group=io.gitlab.jfronny archives_base_name=resclone # Dependencies # check this on https://modmuss50.me/fabric.html -fabric_version=0.32.5+1.16 +fabric_version=0.34.1+1.16 diff --git a/src/main/java/io/gitlab/jfronny/resclone/Resclone.java b/src/main/java/io/gitlab/jfronny/resclone/Resclone.java index daee486..56ce533 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/Resclone.java +++ b/src/main/java/io/gitlab/jfronny/resclone/Resclone.java @@ -9,6 +9,7 @@ import io.gitlab.jfronny.resclone.fetchers.PackFetcher; import io.gitlab.jfronny.resclone.processors.PackProcessor; import io.gitlab.jfronny.resclone.processors.RemoveEmptyProcessor; import io.gitlab.jfronny.resclone.processors.RootPathProcessor; +import io.gitlab.jfronny.resclone.util.PackUrlCache; import io.gitlab.jfronny.resclone.util.Result; import net.fabricmc.api.ModInitializer; import net.fabricmc.loader.api.FabricLoader; @@ -22,6 +23,9 @@ import java.nio.file.FileSystems; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; public class Resclone implements ModInitializer, RescloneApi { @@ -84,19 +88,35 @@ public class Resclone implements ModInitializer, RescloneApi { @Override public void reload() { - //Get paths from patchers. Downloading is implemented in PackFetcher to allow for unconsidered sources Set metas = new LinkedHashSet<>(); - //Download - for (PackMetaUnloaded s : conf) { + try { + ExecutorService pool = Executors.newFixedThreadPool(conf.size()); + for (PackMetaUnloaded s : conf) { + pool.submit(generateTask(s, metas)); + } + pool.shutdown(); + if (!pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS)) { + LOGGER.error("Download timed out. This shouldn't be possible"); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + urlCache.save(); + downloadedPacks.clear(); + downloadedPacks.addAll(metas); + } + + private Runnable generateTask(PackMetaUnloaded meta, Set metas) { + return () -> { try { - if (!fetcherInstances.containsKey(s.fetcher)) - throw new Exception("Invalid fetcher: " + s.fetcher); + if (!fetcherInstances.containsKey(meta.fetcher)) + throw new Exception("Invalid fetcher: " + meta.fetcher); Path cacheDir = getConfigPath().resolve("cache"); PackMetaLoaded p; try { //Download - Result fr = fetcherInstances.get(s.fetcher).get(s.source, cacheDir, s.forceDownload); - p = new PackMetaLoaded(fr.downloadPath, s.name); + Result fr = fetcherInstances.get(meta.fetcher).get(meta.source, cacheDir, meta.forceDownload); + p = new PackMetaLoaded(fr.downloadPath, meta.name); metas.add(p); if (fr.freshDownload) { //Process @@ -116,14 +136,9 @@ public class Resclone implements ModInitializer, RescloneApi { throw new Exception("Failed to download pack", e); } } catch (Throwable e) { - System.err.println("Encountered issue while preparing " + s.name); - e.printStackTrace(); + LOGGER.error("Encountered issue while preparing " + meta.name, e); } - } - //Set variable - urlCache.save(); - downloadedPacks.clear(); - downloadedPacks.addAll(metas); + }; } @Override diff --git a/src/main/java/io/gitlab/jfronny/resclone/fetchers/CurseforgeFetcher.java b/src/main/java/io/gitlab/jfronny/resclone/fetchers/CurseforgeFetcher.java index 5b027fc..b6da1f5 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/fetchers/CurseforgeFetcher.java +++ b/src/main/java/io/gitlab/jfronny/resclone/fetchers/CurseforgeFetcher.java @@ -1,5 +1,6 @@ package io.gitlab.jfronny.resclone.fetchers; +import io.gitlab.jfronny.resclone.Resclone; import io.gitlab.jfronny.resclone.data.CfAddon; import io.gitlab.jfronny.resclone.util.UrlUtils; import net.minecraft.MinecraftVersion; @@ -40,7 +41,7 @@ public class CurseforgeFetcher extends PackFetcher { } if (!foundMatchingVersion) - System.err.println("Could not find pack for matching version, using latest"); + Resclone.LOGGER.error("Could not find pack for matching version, using latest"); return latest.downloadUrl; } catch (Throwable e) { throw new Exception("Could not get CF download for " + baseUrl, e); diff --git a/src/main/java/io/gitlab/jfronny/resclone/processors/PruneVanillaProcessor.java b/src/main/java/io/gitlab/jfronny/resclone/processors/PruneVanillaProcessor.java index 8959aeb..c3168b3 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/processors/PruneVanillaProcessor.java +++ b/src/main/java/io/gitlab/jfronny/resclone/processors/PruneVanillaProcessor.java @@ -1,6 +1,6 @@ package io.gitlab.jfronny.resclone.processors; -import io.gitlab.jfronny.resclone.io.PathPruneVisitor; +import io.gitlab.jfronny.resclone.util.io.PathPruneVisitor; import net.minecraft.client.MinecraftClient; import org.apache.commons.io.IOUtils; diff --git a/src/main/java/io/gitlab/jfronny/resclone/processors/RemoveEmptyProcessor.java b/src/main/java/io/gitlab/jfronny/resclone/processors/RemoveEmptyProcessor.java index 56c04de..bf45657 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/processors/RemoveEmptyProcessor.java +++ b/src/main/java/io/gitlab/jfronny/resclone/processors/RemoveEmptyProcessor.java @@ -1,6 +1,6 @@ package io.gitlab.jfronny.resclone.processors; -import io.gitlab.jfronny.resclone.io.PathPruneVisitor; +import io.gitlab.jfronny.resclone.util.io.PathPruneVisitor; import java.io.IOException; import java.nio.file.DirectoryStream; diff --git a/src/main/java/io/gitlab/jfronny/resclone/processors/RootPathProcessor.java b/src/main/java/io/gitlab/jfronny/resclone/processors/RootPathProcessor.java index 0010391..5fcb2fb 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/processors/RootPathProcessor.java +++ b/src/main/java/io/gitlab/jfronny/resclone/processors/RootPathProcessor.java @@ -1,6 +1,6 @@ package io.gitlab.jfronny.resclone.processors; -import io.gitlab.jfronny.resclone.io.MoveDirVisitor; +import io.gitlab.jfronny.resclone.util.io.MoveDirVisitor; import java.io.IOException; import java.nio.file.*; diff --git a/src/main/java/io/gitlab/jfronny/resclone/PackUrlCache.java b/src/main/java/io/gitlab/jfronny/resclone/util/PackUrlCache.java similarity index 96% rename from src/main/java/io/gitlab/jfronny/resclone/PackUrlCache.java rename to src/main/java/io/gitlab/jfronny/resclone/util/PackUrlCache.java index 5cec41d..df714de 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/PackUrlCache.java +++ b/src/main/java/io/gitlab/jfronny/resclone/util/PackUrlCache.java @@ -1,4 +1,4 @@ -package io.gitlab.jfronny.resclone; +package io.gitlab.jfronny.resclone.util; import java.io.BufferedReader; import java.io.BufferedWriter; diff --git a/src/main/java/io/gitlab/jfronny/resclone/io/MoveDirVisitor.java b/src/main/java/io/gitlab/jfronny/resclone/util/io/MoveDirVisitor.java similarity index 96% rename from src/main/java/io/gitlab/jfronny/resclone/io/MoveDirVisitor.java rename to src/main/java/io/gitlab/jfronny/resclone/util/io/MoveDirVisitor.java index 65f7978..395b504 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/io/MoveDirVisitor.java +++ b/src/main/java/io/gitlab/jfronny/resclone/util/io/MoveDirVisitor.java @@ -1,4 +1,4 @@ -package io.gitlab.jfronny.resclone.io; +package io.gitlab.jfronny.resclone.util.io; import java.io.IOException; import java.nio.file.*; diff --git a/src/main/java/io/gitlab/jfronny/resclone/io/PathPruneVisitor.java b/src/main/java/io/gitlab/jfronny/resclone/util/io/PathPruneVisitor.java similarity index 95% rename from src/main/java/io/gitlab/jfronny/resclone/io/PathPruneVisitor.java rename to src/main/java/io/gitlab/jfronny/resclone/util/io/PathPruneVisitor.java index 76dd6f9..040ac6b 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/io/PathPruneVisitor.java +++ b/src/main/java/io/gitlab/jfronny/resclone/util/io/PathPruneVisitor.java @@ -1,4 +1,4 @@ -package io.gitlab.jfronny.resclone.io; +package io.gitlab.jfronny.resclone.util.io; import java.io.IOException; import java.nio.file.FileVisitResult; diff --git a/src/main/java/io/gitlab/jfronny/resclone/io/RemoveDirVisitor.java b/src/main/java/io/gitlab/jfronny/resclone/util/io/RemoveDirVisitor.java similarity index 93% rename from src/main/java/io/gitlab/jfronny/resclone/io/RemoveDirVisitor.java rename to src/main/java/io/gitlab/jfronny/resclone/util/io/RemoveDirVisitor.java index 1e5c335..f093229 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/io/RemoveDirVisitor.java +++ b/src/main/java/io/gitlab/jfronny/resclone/util/io/RemoveDirVisitor.java @@ -1,4 +1,4 @@ -package io.gitlab.jfronny.resclone.io; +package io.gitlab.jfronny.resclone.util.io; import java.io.IOException; import java.nio.file.FileVisitResult;