From 162fdee877a0d4e7fafd50e24876f46e1bb7fff5 Mon Sep 17 00:00:00 2001 From: JFronny Date: Sun, 31 Jul 2022 14:18:49 +0200 Subject: [PATCH] Implement cache pruning, closes #7 --- .../io/gitlab/jfronny/resclone/Resclone.java | 32 ++++++++++ .../resclone/RescloneEntryDefault.java | 4 +- .../jfronny/resclone/api/RescloneApi.java | 2 + .../util/{ => config}/ConfigLoader.java | 32 ++++------ .../resclone/util/config/RescloneConfig.java | 11 ++++ .../config/RescloneConfigTypeAdapter.java | 64 +++++++++++++++++++ 6 files changed, 126 insertions(+), 19 deletions(-) rename src/main/java/io/gitlab/jfronny/resclone/util/{ => config}/ConfigLoader.java (53%) create mode 100644 src/main/java/io/gitlab/jfronny/resclone/util/config/RescloneConfig.java create mode 100644 src/main/java/io/gitlab/jfronny/resclone/util/config/RescloneConfigTypeAdapter.java diff --git a/src/main/java/io/gitlab/jfronny/resclone/Resclone.java b/src/main/java/io/gitlab/jfronny/resclone/Resclone.java index 7b0649c..43ce449 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/Resclone.java +++ b/src/main/java/io/gitlab/jfronny/resclone/Resclone.java @@ -16,6 +16,8 @@ import java.nio.file.FileSystem; import java.nio.file.*; import java.util.*; import java.util.concurrent.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class Resclone implements ModInitializer, RescloneApi { public static final Set CONF = new LinkedHashSet<>(); @@ -30,6 +32,7 @@ public class Resclone implements ModInitializer, RescloneApi { public static PackUrlCache urlCache; public static int packCount = 0; + private boolean pruneUnused = false; @Override public void onInitialize() { @@ -104,6 +107,8 @@ public class Resclone implements ModInitializer, RescloneApi { urlCache.save(); DOWNLOADED_PACKS.clear(); DOWNLOADED_PACKS.addAll(metas); + + if (pruneUnused) pruneCache(); } private Runnable generateTask(PackMetaUnloaded meta, Set metas) { @@ -142,6 +147,28 @@ public class Resclone implements ModInitializer, RescloneApi { }; } + private void pruneCache() { + Set loadedPacks = DOWNLOADED_PACKS.stream().map(PackMetaLoaded::zipPath).collect(Collectors.toUnmodifiableSet()); + Set toDelete = new HashSet<>(); + try (Stream cacheEntries = Files.list(getConfigPath().resolve("cache"))) { + cacheEntries + .filter(s -> !Files.isRegularFile(s) || !loadedPacks.contains(s)) + .forEach(toDelete::add); + } catch (IOException e) { + LOGGER.error("Could find cache entries to prune", e); + } + if (!toDelete.isEmpty()) { + LOGGER.info("Pruning " + toDelete.size() + " unused cache entries"); + for (Path path : toDelete) { + try { + Files.delete(path); + } catch (IOException e) { + LOGGER.error("Could not delete unused cache entry: " + path, e); + } + } + } + } + @Override public Path getConfigPath() { Path configPath = FabricLoader.getInstance().getConfigDir().resolve("resclone"); @@ -154,4 +181,9 @@ public class Resclone implements ModInitializer, RescloneApi { } return configPath; } + + @Override + public void setPruneUnused(boolean pruneUnused) { + this.pruneUnused = pruneUnused; + } } diff --git a/src/main/java/io/gitlab/jfronny/resclone/RescloneEntryDefault.java b/src/main/java/io/gitlab/jfronny/resclone/RescloneEntryDefault.java index 4c6916f..5fa3f06 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/RescloneEntryDefault.java +++ b/src/main/java/io/gitlab/jfronny/resclone/RescloneEntryDefault.java @@ -1,8 +1,9 @@ package io.gitlab.jfronny.resclone; +import io.gitlab.jfronny.commons.serialize.gson.api.GsonHolder; import io.gitlab.jfronny.resclone.api.RescloneApi; import io.gitlab.jfronny.resclone.api.RescloneEntry; -import io.gitlab.jfronny.resclone.util.ConfigLoader; +import io.gitlab.jfronny.resclone.util.config.*; import io.gitlab.jfronny.resclone.fetchers.BasicFileFetcher; import io.gitlab.jfronny.resclone.fetchers.CurseforgeFetcher; import io.gitlab.jfronny.resclone.fetchers.GitHubFetcher; @@ -13,6 +14,7 @@ import net.fabricmc.loader.api.FabricLoader; public class RescloneEntryDefault implements RescloneEntry { @Override public void init(RescloneApi api) { + GsonHolder.registerTypeAdapter(RescloneConfig.class, new RescloneConfigTypeAdapter()); api.addFetcher(new BasicFileFetcher()); api.addFetcher(new GitHubFetcher()); api.addFetcher(new CurseforgeFetcher()); diff --git a/src/main/java/io/gitlab/jfronny/resclone/api/RescloneApi.java b/src/main/java/io/gitlab/jfronny/resclone/api/RescloneApi.java index 4f956e4..26aa619 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/api/RescloneApi.java +++ b/src/main/java/io/gitlab/jfronny/resclone/api/RescloneApi.java @@ -16,4 +16,6 @@ public interface RescloneApi { void reload(); Path getConfigPath(); + + void setPruneUnused(boolean pruneUnused); } diff --git a/src/main/java/io/gitlab/jfronny/resclone/util/ConfigLoader.java b/src/main/java/io/gitlab/jfronny/resclone/util/config/ConfigLoader.java similarity index 53% rename from src/main/java/io/gitlab/jfronny/resclone/util/ConfigLoader.java rename to src/main/java/io/gitlab/jfronny/resclone/util/config/ConfigLoader.java index ae2a1a8..7021c54 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/util/ConfigLoader.java +++ b/src/main/java/io/gitlab/jfronny/resclone/util/config/ConfigLoader.java @@ -1,11 +1,11 @@ -package io.gitlab.jfronny.resclone.util; +package io.gitlab.jfronny.resclone.util.config; -import com.google.gson.reflect.TypeToken; -import io.gitlab.jfronny.commons.serialize.gson.api.*; +import io.gitlab.jfronny.commons.serialize.gson.api.GsonHolder; import io.gitlab.jfronny.resclone.Resclone; import io.gitlab.jfronny.resclone.api.RescloneApi; import io.gitlab.jfronny.resclone.data.PackMetaUnloaded; +import java.io.BufferedReader; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -16,29 +16,25 @@ public class ConfigLoader { public static void load(RescloneApi api) { Path configPath = api.getConfigPath().resolve("config.json"); if (!Files.exists(configPath)) { - save(api, new HashSet<>()); + save(api, new HashSet<>(), true); } - try { - StringBuilder text = new StringBuilder(); - for (String s : Files.readAllLines(configPath)) { - text.append("\r\n"); - text.append(s); - } - Set data = GsonHolder.getGson().fromJson(text.toString(), new TypeToken>(){}.getType()); - for (PackMetaUnloaded meta : data) { + try (BufferedReader reader = Files.newBufferedReader(configPath)) { + RescloneConfig data = GsonHolder.getGson().fromJson(reader, RescloneConfig.class); + api.setPruneUnused(data.pruneUnused()); + for (PackMetaUnloaded meta : data.packs()) { api.addPack(meta.fetcher, meta.source, meta.name, meta.forceDownload, meta.forceEnable); } + if (data.updateRequired()) { + save(api, data.packs(), data.pruneUnused()); + } } catch (IOException e) { Resclone.LOGGER.error("Could not load config", e); } } - public static void save(RescloneApi api, Set data) { - Path configPath = api.getConfigPath().resolve("config.json"); - Set text = new HashSet<>(); - text.add(GsonHolder.getGson().toJson(data)); - try { - Files.write(configPath, text); + public static void save(RescloneApi api, Set packs, boolean pruneUnused) { + try (var writer = Files.newBufferedWriter(api.getConfigPath().resolve("config.json"))) { + GsonHolder.getGson().toJson(new RescloneConfig(packs, pruneUnused), writer); } catch (IOException e) { Resclone.LOGGER.error("Could not write config", e); } diff --git a/src/main/java/io/gitlab/jfronny/resclone/util/config/RescloneConfig.java b/src/main/java/io/gitlab/jfronny/resclone/util/config/RescloneConfig.java new file mode 100644 index 0000000..191348a --- /dev/null +++ b/src/main/java/io/gitlab/jfronny/resclone/util/config/RescloneConfig.java @@ -0,0 +1,11 @@ +package io.gitlab.jfronny.resclone.util.config; + +import io.gitlab.jfronny.resclone.data.PackMetaUnloaded; + +import java.util.Set; + +public record RescloneConfig(Set packs, boolean pruneUnused, boolean updateRequired) { + public RescloneConfig(Set packs, boolean pruneUnused) { + this(packs, pruneUnused, false); + } +} diff --git a/src/main/java/io/gitlab/jfronny/resclone/util/config/RescloneConfigTypeAdapter.java b/src/main/java/io/gitlab/jfronny/resclone/util/config/RescloneConfigTypeAdapter.java new file mode 100644 index 0000000..8b3041a --- /dev/null +++ b/src/main/java/io/gitlab/jfronny/resclone/util/config/RescloneConfigTypeAdapter.java @@ -0,0 +1,64 @@ +package io.gitlab.jfronny.resclone.util.config; + +import com.google.gson.reflect.TypeToken; +import io.gitlab.jfronny.commons.serialize.gson.api.GsonHolder; +import io.gitlab.jfronny.gson.*; +import io.gitlab.jfronny.gson.stream.*; +import io.gitlab.jfronny.resclone.data.PackMetaUnloaded; + +import java.io.IOException; +import java.lang.reflect.Type; +import java.util.Set; + +public class RescloneConfigTypeAdapter extends TypeAdapter { + private static final String PACKS = "packs"; + private static final String PRUNE_UNUSED = "pruneUnused"; + private static final Type META_SET = new TypeToken>(){}.getType(); + + @Override + public RescloneConfig read(JsonReader jsonReader) throws IOException { + if (jsonReader.peek() == JsonToken.BEGIN_ARRAY) { + // Legacy format compatibility + return new RescloneConfig(GsonHolder.getGson().fromJson(jsonReader, META_SET), false, true); + } else if (jsonReader.peek() == JsonToken.BEGIN_OBJECT) { + // New format + jsonReader.beginObject(); + Set packs = null; + Boolean pruneUnused = null; + boolean updateRequired = false; + while (jsonReader.peek() != JsonToken.END_OBJECT) { + final String name = jsonReader.nextName(); + switch (name) { + case PACKS -> { + if (packs != null) throw new JsonParseException("Unexpected duplicate \"" + PACKS + "\" in Resclone config"); + packs = GsonHolder.getGson().fromJson(jsonReader, META_SET); + } + case PRUNE_UNUSED -> { + if (pruneUnused != null) throw new JsonParseException("Unexpected duplicate \"" + PRUNE_UNUSED + "\" in Resclone config"); + pruneUnused = jsonReader.nextBoolean(); + } + default -> throw new JsonParseException("Unexpected element: \"" + name + "\" in Resclone config"); + } + } + jsonReader.endObject(); + if (packs == null) throw new JsonParseException("Expected Resclone config object to contain packs"); + if (pruneUnused == null) { + pruneUnused = true; + updateRequired = true; + } + return new RescloneConfig(packs, pruneUnused, updateRequired); + } else throw new JsonParseException("Expected Resclone config to be an object or array"); + } + + @Override + public void write(JsonWriter jsonWriter, RescloneConfig rescloneConfig) throws IOException { + jsonWriter.beginObject() + .comment("The packs to be loaded by resclone") + .name(PACKS); + GsonHolder.getGson().toJson(rescloneConfig.packs(), META_SET, jsonWriter); + jsonWriter.comment("Whether to prune unused packs from the cache") + .name(PRUNE_UNUSED) + .value(rescloneConfig.pruneUnused()) + .endObject(); + } +}