1.19.4-pre1 port + modrinth fetcher

README.md

@@ -3,43 +3,65 @@ Packs are updated on startup whenever possible, meaning that very large packs wi
 
 Example config:
 ```
-[
+{
-  {
+  // The packs to be loaded by resclone
-    "fetcher": "file",
+  "packs": [
-    "source": "https://github.com/FaithfulTeam/Faithful/raw/releases/1.16.zip",
+    {
-    "name": "Faithful"
+      "fetcher": "file",
-  },
+      "source": "https://github.com/FaithfulTeam/Faithful/raw/releases/1.16.zip",
-  {
+      "name": "Faithful",
-    "fetcher": "github",
+      "forceDownload": false,
-    "source": "seasnail8169/SnailPack",
+      "forceEnable": false
-    "name": "SnailPack"
+    },
-  },
+    {
-  {
+      "fetcher": "github",
-    "fetcher": "github",
+      "source": "seasnail8169/SnailPack",
-    "source": "spiralhalo/LumiLights/release",
+      "name": "SnailPack",
-    "name": "LumiLights",
+      "forceDownload": false,
-    "forceDownload": true
+      "forceEnable": false
-  },
+    },
-  {
+    {
-    "fetcher": "github",
+      "fetcher": "github",
-    "source": "spiralhalo/LumiPBRExt/tag/v0.7",
+      "source": "spiralhalo/LumiLights/release",
-    "name": "LumiPBRExt"
+      "name": "LumiLights",
-  },
+      "forceDownload": true,
-  {
+      "forceEnable": false
-    "fetcher": "github",
+    },
-    "source": "FaithfulTeam/Faithful/branch/1.16",
+    {
-    "name": "Faithful"
+      "fetcher": "github",
-  },
+      "source": "spiralhalo/LumiPBRExt/tag/v0.7",
-  {
+      "name": "LumiPBRExt",
-    "fetcher": "file",
+      "forceDownload": false,
-    "source": "https://media.forgecdn.net/files/3031/178/BattyCoordinates_005.zip",
+      "forceEnable": false
-    "name": "Battys Coordinates"
+    },
-  },
+    {
-  {
+      "fetcher": "github",
-    "fetcher": "curseforge",
+      "source": "FaithfulTeam/Faithful/branch/1.16",
-    "source": "325017",
+      "name": "Faithful",
-    "name": "Vanilla Additions",
+      "forceDownload": false,
-    "forceEnable": true
+      "forceEnable": false
-  }
+    },
-]
+    {
+      "fetcher": "file",
+      "source": "https://media.forgecdn.net/files/3031/178/BattyCoordinates_005.zip",
+      "name": "Battys Coordinates",
+      "forceDownload": false,
+      "forceEnable": false
+    },
+    {
+      "fetcher": "curseforge",
+      "source": "325017",
+      "name": "Vanilla Additions",
+      "forceDownload": false,
+      "forceEnable": true
+    },
+    {
+      "fetcher": "modrinth",
+      "source": "new-in-town",
+      "name": "New in Town"
+    }
+  ],
+  // Whether to prune unused packs from the cache
+  "pruneUnused": true
+}
 ```

build.gradle.kts

@@ -10,5 +10,5 @@ dependencies {
 
     // Dev env
     modLocalRuntime("io.gitlab.jfronny.libjf:libjf-devutil:${prop("libjf_version")}")
-    modLocalRuntime("com.terraformersmc:modmenu:5.0.2")
+    modLocalRuntime("com.terraformersmc:modmenu:${prop("modmenu_version")}")
 }

gradle.properties

@@ -1,12 +1,13 @@
 # https://fabricmc.net/develop
-minecraft_version=1.19.3
+minecraft_version=1.19.4-pre1
-yarn_mappings=build.5
+yarn_mappings=build.6
-loader_version=0.14.12
+loader_version=0.14.17
 
 maven_group=io.gitlab.jfronny
 archives_base_name=resclone
 
 modrinth_id=resclone
 
-fabric_version=0.70.0+1.19.3
+fabric_version=0.75.1+1.19.4
-libjf_version=3.4.1
+libjf_version=3.5.0-SNAPSHOT
+modmenu_version=6.1.0-beta.3

src/main/java/io/gitlab/jfronny/resclone/Resclone.java

@@ -1,18 +1,19 @@
 package io.gitlab.jfronny.resclone;
 
-import com.google.gson.*;
+import io.gitlab.jfronny.commons.log.Logger;
-import io.gitlab.jfronny.commons.log.*;
+import io.gitlab.jfronny.commons.serialize.gson.api.v1.GsonHolders;
-import io.gitlab.jfronny.commons.serialize.gson.api.*;
 import io.gitlab.jfronny.resclone.api.*;
-import io.gitlab.jfronny.resclone.data.*;
+import io.gitlab.jfronny.resclone.data.PackMetaLoaded;
-import io.gitlab.jfronny.resclone.processors.*;
+import io.gitlab.jfronny.resclone.data.PackMetaUnloaded;
-import io.gitlab.jfronny.resclone.util.*;
+import io.gitlab.jfronny.resclone.processors.RemoveEmptyProcessor;
-import net.fabricmc.api.*;
+import io.gitlab.jfronny.resclone.processors.RootPathProcessor;
-import net.fabricmc.loader.api.*;
+import io.gitlab.jfronny.resclone.util.PackUrlCache;
+import net.fabricmc.api.EnvType;
+import net.fabricmc.api.ModInitializer;
+import net.fabricmc.loader.api.FabricLoader;
 
-import java.io.*;
+import java.io.IOException;
-import java.net.*;
+import java.net.URI;
-import java.nio.file.FileSystem;
 import java.nio.file.*;
 import java.util.*;
 import java.util.concurrent.*;
@@ -28,6 +29,11 @@ public class Resclone implements ModInitializer, RescloneApi {
 
     public static final String MOD_ID = "resclone";
     public static final Logger LOGGER = Logger.forName(MOD_ID);
+    public static final String USER_AGENT = "jfmods/" + MOD_ID + "/" + FabricLoader.getInstance()
+            .getModContainer(MOD_ID).orElseThrow()
+            .getMetadata()
+            .getVersion()
+            .getFriendlyString();
 
     public static PackUrlCache urlCache;
 
@@ -37,7 +43,7 @@ public class Resclone implements ModInitializer, RescloneApi {
     @Override
     public void onInitialize() {
         LOGGER.info("Initialising Resclone.");
-        GsonHolder.register();
+        GsonHolders.registerSerializer();
 
         urlCache = new PackUrlCache(getConfigPath().resolve("urlCache.properties"));
         CONF.clear();

src/main/java/io/gitlab/jfronny/resclone/RescloneEntryDefault.java

@@ -1,23 +1,22 @@
 package io.gitlab.jfronny.resclone;
 
-import io.gitlab.jfronny.commons.serialize.gson.api.GsonHolder;
+import io.gitlab.jfronny.commons.serialize.gson.api.v1.GsonHolders;
 import io.gitlab.jfronny.resclone.api.RescloneApi;
 import io.gitlab.jfronny.resclone.api.RescloneEntry;
-import io.gitlab.jfronny.resclone.util.config.*;
+import io.gitlab.jfronny.resclone.fetchers.*;
-import io.gitlab.jfronny.resclone.fetchers.BasicFileFetcher;
-import io.gitlab.jfronny.resclone.fetchers.CurseforgeFetcher;
-import io.gitlab.jfronny.resclone.fetchers.GitHubFetcher;
 import io.gitlab.jfronny.resclone.processors.PruneVanillaProcessor;
+import io.gitlab.jfronny.resclone.util.config.*;
 import net.fabricmc.api.EnvType;
 import net.fabricmc.loader.api.FabricLoader;
 
 public class RescloneEntryDefault implements RescloneEntry {
     @Override
     public void init(RescloneApi api) {
-        GsonHolder.registerTypeAdapter(RescloneConfig.class, new RescloneConfigTypeAdapter());
+        GsonHolders.registerTypeAdapter(RescloneConfig.class, new RescloneConfigTypeAdapter());
         api.addFetcher(new BasicFileFetcher());
         api.addFetcher(new GitHubFetcher());
         api.addFetcher(new CurseforgeFetcher());
+        api.addFetcher(new ModrinthFetcher());
         if (FabricLoader.getInstance().getEnvironmentType() == EnvType.CLIENT)
             api.addProcessor(new PruneVanillaProcessor());
         ConfigLoader.load(api);

src/main/java/io/gitlab/jfronny/resclone/data/modrinth/Version.java

@@ -0,0 +1,64 @@
+package io.gitlab.jfronny.resclone.data.modrinth;
+
+import io.gitlab.jfronny.gson.annotations.SerializedName;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.Date;
+import java.util.List;
+
+public class Version {
+    public String name;
+    public String version_number;
+    @Nullable public String changelog;
+    public List<Dependency> dependencies;
+    public List<String> game_versions;
+    public VersionType version_type;
+    public List<String> loaders;
+    public Boolean featured;
+    public Status status;
+    @Nullable public Status requested_status;
+    public String id;
+    public String project_id;
+    public String author_id;
+    public Date date_published;
+    public Integer downloads;
+    public List<File> files;
+
+    public static class Dependency {
+        @Nullable public String version_id;
+        @Nullable public String project_id;
+        @Nullable public String file_name;
+        public Type dependency_type;
+
+        public enum Type {
+            required, optional, incompatible, embedded
+        }
+    }
+
+    public enum VersionType {
+        release, beta, alpha
+    }
+
+    public enum Status {
+        listed, archived, draft, unlisted, scheduled, unknown
+    }
+
+    public static class File {
+        public Hashes hashes;
+        public String url;
+        public String filename;
+        public Boolean primary;
+        public Integer size;
+        @Nullable public Type file_type;
+
+        public static class Hashes {
+            public String sha512;
+            public String sha1;
+        }
+
+        public enum Type {
+            @SerializedName("required-resource-pack") REQUIRED_RESOURCE_PACK,
+            @SerializedName("optional-resource-pack") OPTIONAL_RESOURCE_PACK
+        }
+    }
+}

src/main/java/io/gitlab/jfronny/resclone/fetchers/BasePackFetcher.java

@@ -1,11 +1,10 @@
 package io.gitlab.jfronny.resclone.fetchers;
 
+import io.gitlab.jfronny.commons.HttpUtils;
 import io.gitlab.jfronny.resclone.Resclone;
 import io.gitlab.jfronny.resclone.api.PackFetcher;
 
-import java.io.FileOutputStream;
+import java.io.*;
-import java.io.InputStream;
-import java.net.URL;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
@@ -34,8 +33,8 @@ public abstract class BasePackFetcher implements PackFetcher {
 
         Resclone.LOGGER.info("Downloading pack: " + url);
 
-        try (InputStream is = new URL(url).openStream()) {
+        try (InputStream is = HttpUtils.get(url).userAgent(Resclone.USER_AGENT).sendInputStream();
-            FileOutputStream os = new FileOutputStream(p.toFile());
+             OutputStream os = Files.newOutputStream(p)) {
             byte[] dataBuffer = new byte[1024];
             int bytesRead;
             while ((bytesRead = is.read(dataBuffer, 0, 1024)) != -1) {
@@ -43,7 +42,7 @@ public abstract class BasePackFetcher implements PackFetcher {
             }
             Resclone.LOGGER.info("Finished downloading.");
         } catch (Throwable e) {
-            throw new Exception("Could not download pack", e);
+            throw new IOException("Could not download pack", e);
         }
         Resclone.packCount++;
         return new Result(p, true);

src/main/java/io/gitlab/jfronny/resclone/fetchers/CurseforgeFetcher.java

@@ -1,13 +1,16 @@
 package io.gitlab.jfronny.resclone.fetchers;
 
-import io.gitlab.jfronny.commons.*;
+import io.gitlab.jfronny.commons.HttpUtils;
-import io.gitlab.jfronny.resclone.*;
+import io.gitlab.jfronny.resclone.Resclone;
-import io.gitlab.jfronny.resclone.data.curseforge.*;
+import io.gitlab.jfronny.resclone.data.curseforge.GetModFilesResponse;
-import net.minecraft.*;
+import io.gitlab.jfronny.resclone.data.curseforge.GetModResponse;
+import net.minecraft.MinecraftVersion;
 
-import java.io.*;
+import java.io.FileNotFoundException;
-import java.net.*;
+import java.io.IOException;
-import java.util.*;
+import java.net.URISyntaxException;
+import java.util.Date;
+import java.util.Random;
 
 public class CurseforgeFetcher extends BasePackFetcher {
     // So you found the API key.
@@ -41,8 +44,7 @@ public class CurseforgeFetcher extends BasePackFetcher {
             boolean foundMatchingVersion = false;
 
             for (GetModFilesResponse.Data addon : GET(baseUrl + "/files", GetModFilesResponse.class).data) {
-                if (foundMatchingVersion && !addon.gameVersions.contains(version))
+                if (foundMatchingVersion && !addon.gameVersions.contains(version)) continue;
-                    continue;
                 if (!foundMatchingVersion && addon.gameVersions.contains(version)) {
                     foundMatchingVersion = true;
                     latest = null;
@@ -53,11 +55,11 @@ public class CurseforgeFetcher extends BasePackFetcher {
                 }
             }
 
-            if (!foundMatchingVersion)
+            if (latest == null) throw new FileNotFoundException("Could not identify valid version");
-                Resclone.LOGGER.error("Could not find pack for matching version, using latest");
+            if (!foundMatchingVersion) Resclone.LOGGER.error("Could not find matching version of " + baseUrl + ", using latest");
             return latest.downloadUrl;
         } catch (Throwable e) {
-            throw new Exception("Could not get CF download for " + baseUrl, e);
+            throw new IOException("Could not get CurseForge download for " + baseUrl, e);
         }
     }
 

src/main/java/io/gitlab/jfronny/resclone/fetchers/ModrinthFetcher.java

@@ -0,0 +1,62 @@
+package io.gitlab.jfronny.resclone.fetchers;
+
+import com.google.common.reflect.TypeToken;
+import io.gitlab.jfronny.commons.HttpUtils;
+import io.gitlab.jfronny.resclone.Resclone;
+import io.gitlab.jfronny.resclone.data.modrinth.Version;
+import net.minecraft.MinecraftVersion;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import java.util.Date;
+import java.util.List;
+
+public class ModrinthFetcher extends BasePackFetcher {
+    private static final Type versionType = new TypeToken<List<Version>>() {}.getType();
+    @Override
+    public String getSourceTypeName() {
+        return "modrinth";
+    }
+
+    @Override
+    public String getDownloadUrl(String baseUrl) throws Exception {
+        try {
+            String version = MinecraftVersion.CURRENT.getName();
+
+            Version latest = null;
+            Date latestDate = null;
+            boolean foundMatchingVersion = false;
+
+            for (Version ver : HttpUtils.get("https://api.modrinth.com/v2/project/" + baseUrl + "/version")
+                    .userAgent(Resclone.USER_AGENT)
+                    .<List<Version>>sendSerialized(versionType)) {
+                if (foundMatchingVersion && !ver.game_versions.contains(version)) continue;
+                if (ver.files.isEmpty()) continue;
+                if (!foundMatchingVersion && ver.game_versions.contains(version)) {
+                    foundMatchingVersion = true;
+                    latest = null;
+                }
+                if (latest == null || ver.date_published.after(latestDate)) {
+                    latest = ver;
+                    latestDate = ver.date_published;
+                }
+            }
+
+            if (latest == null) throw new FileNotFoundException("Could not identify valid version");
+            if (!foundMatchingVersion) Resclone.LOGGER.error("Could not find matching version of " + baseUrl + ", using latest");
+
+            for (Version.File file : latest.files) {
+                if (file.primary) return file.url;
+            }
+            Resclone.LOGGER.error("Could not identify primary file of " + baseUrl + ", attempting identification by file_type");
+            for (Version.File file : latest.files) {
+                if (file.file_type == Version.File.Type.REQUIRED_RESOURCE_PACK) return file.url;
+            }
+            Resclone.LOGGER.error("Identification failed, using first file of " + baseUrl);
+            return latest.files.get(0).url;
+        } catch (Throwable e) {
+            throw new IOException("Could not get Modrinth download for " + baseUrl, e);
+        }
+    }
+}

src/main/java/io/gitlab/jfronny/resclone/util/config/ConfigLoader.java

@@ -1,6 +1,6 @@
 package io.gitlab.jfronny.resclone.util.config;
 
-import io.gitlab.jfronny.commons.serialize.gson.api.GsonHolder;
+import io.gitlab.jfronny.commons.serialize.gson.api.v1.GsonHolders;
 import io.gitlab.jfronny.resclone.Resclone;
 import io.gitlab.jfronny.resclone.api.RescloneApi;
 import io.gitlab.jfronny.resclone.data.PackMetaUnloaded;
@@ -19,7 +19,7 @@ public class ConfigLoader {
             save(api, new HashSet<>(), true);
         }
         try (BufferedReader reader = Files.newBufferedReader(configPath)) {
-            RescloneConfig data = GsonHolder.getGson().fromJson(reader, RescloneConfig.class);
+            RescloneConfig data = GsonHolders.CONFIG.getGson().fromJson(reader, RescloneConfig.class);
             api.setPruneUnused(data.pruneUnused());
             for (PackMetaUnloaded meta : data.packs()) {
                 api.addPack(meta.fetcher, meta.source, meta.name, meta.forceDownload, meta.forceEnable);
@@ -34,7 +34,7 @@ public class ConfigLoader {
 
     public static void save(RescloneApi api, Set<PackMetaUnloaded> packs, boolean pruneUnused) {
         try (var writer = Files.newBufferedWriter(api.getConfigPath().resolve("config.json"))) {
-            GsonHolder.getGson().toJson(new RescloneConfig(packs, pruneUnused), writer);
+            GsonHolders.CONFIG.getGson().toJson(new RescloneConfig(packs, pruneUnused), writer);
         } catch (IOException e) {
             Resclone.LOGGER.error("Could not write config", e);
         }

src/main/java/io/gitlab/jfronny/resclone/util/config/RescloneConfigTypeAdapter.java

@@ -1,8 +1,9 @@
 package io.gitlab.jfronny.resclone.util.config;
 
 import com.google.gson.reflect.TypeToken;
-import io.gitlab.jfronny.commons.serialize.gson.api.GsonHolder;
+import io.gitlab.jfronny.commons.serialize.gson.api.v1.GsonHolders;
-import io.gitlab.jfronny.gson.*;
+import io.gitlab.jfronny.gson.JsonParseException;
+import io.gitlab.jfronny.gson.TypeAdapter;
 import io.gitlab.jfronny.gson.stream.*;
 import io.gitlab.jfronny.resclone.data.PackMetaUnloaded;
 
@@ -19,7 +20,7 @@ public class RescloneConfigTypeAdapter extends TypeAdapter<RescloneConfig> {
     public RescloneConfig read(JsonReader jsonReader) throws IOException {
         if (jsonReader.peek() == JsonToken.BEGIN_ARRAY) {
             // Legacy format compatibility
-            return new RescloneConfig(GsonHolder.getGson().fromJson(jsonReader, META_SET), false, true);
+            return new RescloneConfig(GsonHolders.CONFIG.getGson().fromJson(jsonReader, META_SET), false, true);
         } else if (jsonReader.peek() == JsonToken.BEGIN_OBJECT) {
             // New format
             jsonReader.beginObject();
@@ -31,7 +32,7 @@ public class RescloneConfigTypeAdapter extends TypeAdapter<RescloneConfig> {
                 switch (name) {
                     case PACKS -> {
                         if (packs != null) throw new JsonParseException("Unexpected duplicate \"" + PACKS + "\" in Resclone config");
-                        packs = GsonHolder.getGson().fromJson(jsonReader, META_SET);
+                        packs = GsonHolders.CONFIG.getGson().fromJson(jsonReader, META_SET);
                     }
                     case PRUNE_UNUSED -> {
                         if (pruneUnused != null) throw new JsonParseException("Unexpected duplicate \"" + PRUNE_UNUSED + "\" in Resclone config");
@@ -55,7 +56,7 @@ public class RescloneConfigTypeAdapter extends TypeAdapter<RescloneConfig> {
         jsonWriter.beginObject()
                 .comment("The packs to be loaded by resclone")
                 .name(PACKS);
-        GsonHolder.getGson().toJson(rescloneConfig.packs(), META_SET, jsonWriter);
+        GsonHolders.CONFIG.getGson().toJson(rescloneConfig.packs(), META_SET, jsonWriter);
         jsonWriter.comment("Whether to prune unused packs from the cache")
                 .name(PRUNE_UNUSED)
                 .value(rescloneConfig.pruneUnused())