From 239a8ff8562aadb64f619a14b5b10d8ca4ec07dd Mon Sep 17 00:00:00 2001 From: JFronny <33260128+jfronny@users.noreply.github.com> Date: Wed, 3 Mar 2021 08:05:21 +0100 Subject: [PATCH] If the resolved download url stayes the same, use the downloaded file. If it can't be resolved, use a cached entry --- .../gitlab/jfronny/resclone/PackUrlCache.java | 44 +++++++++++++++ .../io/gitlab/jfronny/resclone/Resclone.java | 54 ++++++++----------- .../resclone/fetchers/PackFetcher.java | 34 ++++++++++-- 3 files changed, 96 insertions(+), 36 deletions(-) create mode 100644 src/main/java/io/gitlab/jfronny/resclone/PackUrlCache.java diff --git a/src/main/java/io/gitlab/jfronny/resclone/PackUrlCache.java b/src/main/java/io/gitlab/jfronny/resclone/PackUrlCache.java new file mode 100644 index 0000000..7f1bbc2 --- /dev/null +++ b/src/main/java/io/gitlab/jfronny/resclone/PackUrlCache.java @@ -0,0 +1,44 @@ +package io.gitlab.jfronny.resclone; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; + +public class PackUrlCache { + Properties properties = new Properties(); + private Path file; + + public PackUrlCache(Path file) { + this.file = file; + if (Files.exists(file)) { + try (BufferedReader r = Files.newBufferedReader(file)) { + properties.load(r); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + public void save() { + try (BufferedWriter w = Files.newBufferedWriter(file)) { + properties.store(w, "This is an internal file used for offline pack loading, do not edit"); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public boolean containsKey(String key) { + return properties.containsKey(key); + } + + public String get(String key) { + return properties.getProperty(key); + } + + public void set(String key, String value) { + properties.setProperty(key, value); + } +} diff --git a/src/main/java/io/gitlab/jfronny/resclone/Resclone.java b/src/main/java/io/gitlab/jfronny/resclone/Resclone.java index f1e0f68..0d40606 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/Resclone.java +++ b/src/main/java/io/gitlab/jfronny/resclone/Resclone.java @@ -28,10 +28,12 @@ public class Resclone implements ModInitializer, RescloneApi { public static final Set downloadedPacks = new LinkedHashSet<>(); public static final Gson gson = new Gson(); public static final String MOD_ID = "resclone"; + public static PackUrlCache urlCache; @Override public void onInitialize() { System.out.println("[resclone] Beginning init, will download packs"); + urlCache = new PackUrlCache(getConfigPath().resolve("urlCache.properties")); conf.clear(); fetcherInstances.clear(); processors.clear(); @@ -67,58 +69,44 @@ public class Resclone implements ModInitializer, RescloneApi { @Override public void reload() { //Get paths from patchers. Downloading is implemented in PackFetcher to allow for unconsidered sources - //TODO implement downloading again after a set time if downloadable Set metas = new LinkedHashSet<>(); //Download for (PackMetaUnloaded s : conf) { try { if (!fetcherInstances.containsKey(s.fetcher)) throw new RescloneException("Invalid fetcher: " + s.fetcher); - String fileName = ""; - fileName += Integer.toHexString(s.fetcher.hashCode()); - fileName += Integer.toHexString(s.source.hashCode()); - PackMetaLoaded p = new PackMetaLoaded(getConfigPath().resolve("cache").resolve(fileName), s.name); - //If file is not present: download and process pack - Path p1 = p.zipPath; - if (Files.exists(p.zipPath)) { - p1 = p.zipPath.getParent().resolve(p.zipPath.getFileName() + ".bak"); - Files.deleteIfExists(p1); - Files.move(p.zipPath, p1); - } + Path cacheDir = getConfigPath().resolve("cache"); + PackMetaLoaded p; try { //Download - fetcherInstances.get(s.fetcher).get(s.source, p.zipPath); - //Process - Map props = new HashMap<>(); - props.put("create", "false"); - URI zipfile = URI.create("jar:" + p.zipPath.toUri().toString()); - try (FileSystem zipfs = FileSystems.newFileSystem(zipfile, props)) { - for (PackProcessor processor : processors) { - processor.process(zipfs); - } + PackFetcher.FetcherResult fr = fetcherInstances.get(s.fetcher).get(s.source, cacheDir); + p = new PackMetaLoaded(fr.downloadPath, s.name); + metas.add(p); + if (fr.freshDownload) { + //Process + Map props = new HashMap<>(); + props.put("create", "false"); + URI zipfile = URI.create("jar:" + p.zipPath.toUri().toString()); + try (FileSystem zipfs = FileSystems.newFileSystem(zipfile, props)) { + for (PackProcessor processor : processors) { + processor.process(zipfs); + } - } catch (Throwable e) { - e.printStackTrace(); + } catch (Throwable e) { + e.printStackTrace(); + } } } catch (Throwable e) { - if (!p1.equals(p.zipPath)) { - e.printStackTrace(); - System.err.println("Failed to download, using cache"); - Files.deleteIfExists(p.zipPath); - Files.move(p1, p.zipPath); - } else - throw new RescloneException("Failed to download initial", e); + throw new RescloneException("Failed to download pack", e); } - if (!p1.equals(p.zipPath)) - Files.deleteIfExists(p1); - metas.add(p); } catch (Throwable e) { System.err.println("Encountered issue while preparing " + s.name); e.printStackTrace(); } } //Set variable + urlCache.save(); downloadedPacks.clear(); downloadedPacks.addAll(metas); } diff --git a/src/main/java/io/gitlab/jfronny/resclone/fetchers/PackFetcher.java b/src/main/java/io/gitlab/jfronny/resclone/fetchers/PackFetcher.java index 11d1f93..86855f0 100644 --- a/src/main/java/io/gitlab/jfronny/resclone/fetchers/PackFetcher.java +++ b/src/main/java/io/gitlab/jfronny/resclone/fetchers/PackFetcher.java @@ -11,6 +11,7 @@ import java.io.InputStream; import java.net.HttpURLConnection; import java.net.URL; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Scanner; import java.util.Set; @@ -47,11 +48,27 @@ public abstract class PackFetcher { return gson.fromJson(readStringFromURL(requestUrl), TypeToken.getParameterized(Set.class, classOfT).getType()); } - public void get(String baseUrl, Path targetPath) throws RescloneException { - String url = getDownloadUrl(baseUrl); + public FetcherResult get(String baseUrl, Path targetDir) throws RescloneException { + String url; + try { + url = getDownloadUrl(baseUrl); + Resclone.urlCache.set(baseUrl, url); + } + catch (RescloneException e) { + if (Resclone.urlCache.containsKey(baseUrl)) { + e.printStackTrace(); + url = Resclone.urlCache.get(baseUrl); + } + else { + throw e; + } + } + Path p = targetDir.resolve(Integer.toHexString(url.hashCode())); + if (Files.exists(p)) + return new FetcherResult(p, false); System.out.println("Downloading pack " + url); try (InputStream is = new URL(url).openStream()) { - FileOutputStream os = new FileOutputStream(targetPath.toFile()); + FileOutputStream os = new FileOutputStream(p.toFile()); byte[] dataBuffer = new byte[1024]; int bytesRead; while ((bytesRead = is.read(dataBuffer, 0, 1024)) != -1) { @@ -62,5 +79,16 @@ public abstract class PackFetcher { catch (Throwable e) { throw new RescloneException("Could not download pack", e); } + return new FetcherResult(p, true); + } + + public class FetcherResult { + public final Path downloadPath; + public final boolean freshDownload; + + public FetcherResult(Path downloadPath, boolean freshDownload) { + this.downloadPath = downloadPath; + this.freshDownload = freshDownload; + } } }