diff --git a/build.gradle.kts b/build.gradle.kts index 2558ba6..d89f3b9 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - java + `java-library` application id("com.github.johnrengelman.shadow") version "7.1.2" } @@ -13,16 +13,23 @@ application { repositories { mavenCentral() - maven("https://gitlab.com/api/v4/projects/35745143/packages/maven") + maven("https://maven.frohnmeyer-wds.de/artifacts") } dependencies { implementation("net.freeutils:jlhttp:2.6") - implementation("com.amihaiemil.web:eo-yaml:6.1.0") - implementation("io.gitlab.jfronny:commons:2022.9.26+17-56-16") - implementation("io.gitlab.jfronny:commons-gson:2022.9.26+17-56-16") + implementation("io.gitlab.jfronny:commons:2022.11.2+21-36-56") + implementation("io.gitlab.jfronny:commons-gson:2022.11.2+21-36-56") + + api("io.gitlab.jfronny.gson:gson-compile-core:1.0-SNAPSHOT") + compileOnly("io.gitlab.jfronny.gson:gson-compile-annotations:1.0-SNAPSHOT") + annotationProcessor("io.gitlab.jfronny.gson:gson-compile-processor:1.0-SNAPSHOT") } tasks.runShadow { args("8002") } + +tasks.withType { + options.compilerArgs.add("-AgsonCompileNoReflect") +} \ No newline at end of file diff --git a/src/main/java/io/gitlab/jfronny/woodpecker/include/Main.java b/src/main/java/io/gitlab/jfronny/woodpecker/include/Main.java index a80e2fd..2863ef6 100644 --- a/src/main/java/io/gitlab/jfronny/woodpecker/include/Main.java +++ b/src/main/java/io/gitlab/jfronny/woodpecker/include/Main.java @@ -1,12 +1,8 @@ package io.gitlab.jfronny.woodpecker.include; -import com.amihaiemil.eoyaml.*; import io.gitlab.jfronny.commons.HttpUtils; import io.gitlab.jfronny.commons.log.Logger; -import io.gitlab.jfronny.commons.serialize.Serializer; -import io.gitlab.jfronny.commons.serialize.gson.api.v1.GsonHolders; import io.gitlab.jfronny.woodpecker.include.model.*; -import io.gitlab.jfronny.woodpecker.include.util.YamlTypeAdapter; import net.freeutils.httpserver.HTTPServer; import java.io.*; @@ -22,8 +18,6 @@ public class Main { } HTTPServer server = new HTTPServer(Integer.parseInt(args[0])); HTTPServer.VirtualHost host = server.getVirtualHost(null); - GsonHolders.registerSerializer(); - GsonHolders.registerTypeAdapter(YamlMapping.class, new YamlTypeAdapter()); HttpUtils.setUserAgent("Woodpecker-Include/1.0"); host.addContext("/ciconfig", (req, resp) -> { Ref ref = new Ref(); @@ -45,9 +39,8 @@ public class Main { private static int processRequest(HTTPServer.Request req, HTTPServer.Response resp, Ref ref) throws IOException { RequestModel request; - try (var is = req.getBody()) { - String body = new String(is.readAllBytes()); - request = Serializer.getInstance().deserialize(body, RequestModel.class); + try (var isr = new InputStreamReader(req.getBody())) { + request = GC_RequestModel.read(isr); } ref.name = request.repo.full_name; AtomicBoolean changed = new AtomicBoolean(false); @@ -55,7 +48,7 @@ public class Main { if (changed.get()) { resp.sendHeaders(200); try (OutputStreamWriter writer = new OutputStreamWriter(resp.getBody())) { - Serializer.getInstance().serialize(response, writer); + GC_ResponseModel.write(writer, response); } LOG.info("Processed includes for pipeline for " + ref.name); } else { diff --git a/src/main/java/io/gitlab/jfronny/woodpecker/include/PipelineUnpacker.java b/src/main/java/io/gitlab/jfronny/woodpecker/include/PipelineUnpacker.java index 40c2507..2b19f66 100644 --- a/src/main/java/io/gitlab/jfronny/woodpecker/include/PipelineUnpacker.java +++ b/src/main/java/io/gitlab/jfronny/woodpecker/include/PipelineUnpacker.java @@ -1,9 +1,8 @@ package io.gitlab.jfronny.woodpecker.include; -import com.amihaiemil.eoyaml.*; import io.gitlab.jfronny.commons.HttpUtils; +import io.gitlab.jfronny.commons.StreamIterable; import io.gitlab.jfronny.woodpecker.include.model.Pipeline; -import io.gitlab.jfronny.woodpecker.include.util.FilteringYamlMapping; import java.io.*; import java.net.URI; @@ -11,12 +10,17 @@ import java.net.URISyntaxException; import java.nio.file.Paths; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.BiConsumer; -import java.util.function.Consumer; +import java.util.function.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; final class PipelineUnpacker implements BiConsumer> { + private static final String URL = "https?://(?:www\\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b[-a-zA-Z0-9()@:%_+.~#?&/=]*"; + private static final Pattern INCLUDE = Pattern.compile("^#include (" + URL + ")$"); + private static final Pattern LINK = Pattern.compile("^#link (" + URL + ")$"); + private final AtomicBoolean changed; - private final Set included = new HashSet<>(); + private final Set linked = new HashSet<>(); public PipelineUnpacker(AtomicBoolean changed) { this.changed = changed; @@ -25,7 +29,7 @@ final class PipelineUnpacker implements BiConsumer> @Override public void accept(Pipeline pipeline, Consumer pipelineConsumer) { try { - processPipeline(pipeline, pipelineConsumer); + processPipeline(pipeline, pipelineConsumer, 0); } catch (URISyntaxException e) { throw new UncheckedIOException(new IOException("Could not find URL", e)); } catch (IOException e) { @@ -33,47 +37,59 @@ final class PipelineUnpacker implements BiConsumer> } } - private void processPipeline(Pipeline pipeline, Consumer pipelineConsumer) throws URISyntaxException, IOException { - YamlNode include = pipeline.data.value("include"); - if (include == null) { + private void processPipeline(Pipeline pipeline, Consumer pipelineConsumer, int depth) throws URISyntaxException, IOException { + if (depth > 5) throw new IOException("Too many nested includes, a maximum of 5 is supported"); + List toLink = pipeline.data().lines() + .map(LINK::matcher) + .filter(Matcher::matches) + .map(s -> s.group(1)) + .filter(s -> !linked.contains(s)) + .toList(); + boolean hasIncludes = pipeline.data().lines().anyMatch(INCLUDE.asPredicate()); + if (toLink.isEmpty() && !hasIncludes) { // Has no includes pipelineConsumer.accept(pipeline); return; } changed.set(true); - // Send included - //TODO switch back to switch pattern matching once that is available - if (include instanceof Scalar str) download(str.value(), null, pipelineConsumer); - else if (include instanceof YamlMapping map) { - for (YamlNode key : map.keys()) { - download(map.string(key), key.asScalar().value(), pipelineConsumer); + + // Fill in includes and reprocess + if (hasIncludes) { + StringBuilder newData = new StringBuilder(); + for (String line : new StreamIterable<>(pipeline.data().lines())) { + Matcher matcher = INCLUDE.matcher(line); + if (!matcher.matches()) newData.append(line); + else { + newData.append(download(matcher.group(1)).data()); + } + newData.append('\n'); } - } else if (include instanceof YamlSequence seq) { - for (YamlNode node : seq) { - download(node.asScalar().value(), null, pipelineConsumer); - } - } else throw new IllegalStateException("Unexpected value: " + include); - if (pipeline.data.keys().size() > 1) { + processPipeline(new Pipeline(pipeline.name(), newData.toString()), pipelineConsumer, depth + 1); + return; + } + + // Link additional pipelines + for (String url : toLink) { + if (linked.contains(url)) continue; + linked.add(url); + processPipeline(download(url), pipelineConsumer, depth + 1); + } + + if (pipeline.data().lines() + .anyMatch(INCLUDE.asPredicate().negate() + .and(LINK.asPredicate().negate()) + .and(Predicate.not(String::isBlank)))) { // More than just includes: generate override without include node - Pipeline filtered = new Pipeline(); - filtered.name = pipeline.name; - pipeline.data = new FilteringYamlMapping(pipeline.data, s -> !s.asScalar().value().equals("include")); - pipelineConsumer.accept(filtered); + pipelineConsumer.accept(pipeline); } } - private void download(String url, String fileName, Consumer pipelineConsumer) throws URISyntaxException, IOException { - if (included.contains(url)) return; - included.add(url); + private Pipeline download(String url) throws URISyntaxException, IOException { URI uri = new URI(url.replace(" ", "%20")); if (!"http".equalsIgnoreCase(uri.getScheme()) && !"https".equalsIgnoreCase(uri.getScheme())) { throw new URISyntaxException(url, "Could not find scheme"); } - try (InputStream is = HttpUtils.get(url).sendInputStream()) { - Pipeline pipeline = new Pipeline(); - pipeline.name = fileName != null ? fileName : Paths.get(new URI(url.replace(" ", "%20")).getPath()).getFileName().toString(); - pipeline.data = Yaml.createYamlInput(is).readYamlMapping(); - processPipeline(pipeline, pipelineConsumer); - } + String fileName = Paths.get(new URI(url.replace(" ", "%20")).getPath()).getFileName().toString(); + return new Pipeline(fileName, HttpUtils.get(url).sendString()); } } diff --git a/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Build.java b/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Build.java index 3044811..6b0ff7e 100644 --- a/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Build.java +++ b/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Build.java @@ -1,7 +1,10 @@ package io.gitlab.jfronny.woodpecker.include.model; +import io.gitlab.jfronny.gson.compile.annotations.GSerializable; + import java.util.List; +@GSerializable public class Build { public Long id; public Long number; diff --git a/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Pipeline.java b/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Pipeline.java index b49bc65..7b79fb9 100644 --- a/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Pipeline.java +++ b/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Pipeline.java @@ -1,21 +1,7 @@ package io.gitlab.jfronny.woodpecker.include.model; -import com.amihaiemil.eoyaml.YamlMapping; +import io.gitlab.jfronny.gson.compile.annotations.GSerializable; -public class Pipeline { - public String name; - public YamlMapping data; - - @Override - public String toString() { - return "{\"name\":\"" + escape(name) + ",\"data\":\"" + escape(data.toString()) + '}'; - } - - private static String escape(String text) { - return text - .replace("\\", "\\\\") - .replace("\"", "\\\"") - .replace("\r", "\\r") - .replace("\n", "\\n"); - } +@GSerializable +public record Pipeline(String name, String data) { } diff --git a/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Repo.java b/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Repo.java index 7729508..59f4854 100644 --- a/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Repo.java +++ b/src/main/java/io/gitlab/jfronny/woodpecker/include/model/Repo.java @@ -1,9 +1,11 @@ package io.gitlab.jfronny.woodpecker.include.model; import io.gitlab.jfronny.gson.annotations.SerializedName; +import io.gitlab.jfronny.gson.compile.annotations.GSerializable; import java.util.List; +@GSerializable public class Repo { public Long id; public String owner; diff --git a/src/main/java/io/gitlab/jfronny/woodpecker/include/model/RequestModel.java b/src/main/java/io/gitlab/jfronny/woodpecker/include/model/RequestModel.java index 4f1c6da..85902f0 100644 --- a/src/main/java/io/gitlab/jfronny/woodpecker/include/model/RequestModel.java +++ b/src/main/java/io/gitlab/jfronny/woodpecker/include/model/RequestModel.java @@ -1,7 +1,10 @@ package io.gitlab.jfronny.woodpecker.include.model; +import io.gitlab.jfronny.gson.compile.annotations.GSerializable; + import java.util.List; +@GSerializable public class RequestModel { public Repo repo; public Build pipeline; diff --git a/src/main/java/io/gitlab/jfronny/woodpecker/include/model/ResponseModel.java b/src/main/java/io/gitlab/jfronny/woodpecker/include/model/ResponseModel.java index d557a3a..a145fbb 100644 --- a/src/main/java/io/gitlab/jfronny/woodpecker/include/model/ResponseModel.java +++ b/src/main/java/io/gitlab/jfronny/woodpecker/include/model/ResponseModel.java @@ -1,27 +1,9 @@ package io.gitlab.jfronny.woodpecker.include.model; +import io.gitlab.jfronny.gson.compile.annotations.GSerializable; + import java.util.List; -public class ResponseModel { - public List configs; - - public ResponseModel() { - } - - public ResponseModel(List configs) { - this.configs = List.copyOf(configs); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder("{\"configs\":["); - boolean first = false; - for (Pipeline config : configs) { - if (!first) sb.append(','); - first = true; - sb.append(config.toString()); - } - sb.append("]}"); - return sb.toString(); - } +@GSerializable +public record ResponseModel(List configs) { } diff --git a/src/main/java/io/gitlab/jfronny/woodpecker/include/util/FilteringYamlMapping.java b/src/main/java/io/gitlab/jfronny/woodpecker/include/util/FilteringYamlMapping.java deleted file mode 100644 index 768bec7..0000000 --- a/src/main/java/io/gitlab/jfronny/woodpecker/include/util/FilteringYamlMapping.java +++ /dev/null @@ -1,32 +0,0 @@ -package io.gitlab.jfronny.woodpecker.include.util; - -import com.amihaiemil.eoyaml.*; - -import java.util.Set; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -public class FilteringYamlMapping extends BaseYamlMapping { - private final YamlMapping base; - private final Predicate filter; - - public FilteringYamlMapping(YamlMapping base, Predicate filter) { - this.base = base; - this.filter = filter; - } - - @Override - public Set keys() { - return base.keys().stream().filter(filter).collect(Collectors.toSet()); - } - - @Override - public YamlNode value(YamlNode key) { - return filter.test(key) ? base.value(key) : null; - } - - @Override - public Comment comment() { - return base.comment(); - } -} diff --git a/src/main/java/io/gitlab/jfronny/woodpecker/include/util/YamlTypeAdapter.java b/src/main/java/io/gitlab/jfronny/woodpecker/include/util/YamlTypeAdapter.java deleted file mode 100644 index 5fdb711..0000000 --- a/src/main/java/io/gitlab/jfronny/woodpecker/include/util/YamlTypeAdapter.java +++ /dev/null @@ -1,21 +0,0 @@ -package io.gitlab.jfronny.woodpecker.include.util; - -import com.amihaiemil.eoyaml.Yaml; -import com.amihaiemil.eoyaml.YamlMapping; -import io.gitlab.jfronny.gson.TypeAdapter; -import io.gitlab.jfronny.gson.stream.JsonReader; -import io.gitlab.jfronny.gson.stream.JsonWriter; - -import java.io.IOException; - -public class YamlTypeAdapter extends TypeAdapter { - @Override - public void write(JsonWriter jsonWriter, YamlMapping yamlMapping) throws IOException { - jsonWriter.value(yamlMapping.toString()); - } - - @Override - public YamlMapping read(JsonReader jsonReader) throws IOException { - return Yaml.createYamlInput(jsonReader.nextString()).readYamlMapping(); - } -}