Johannes
/
Woodpecker-Include
1
0
Fork 0
Code Issues Pull Requests Packages Releases Activity

Fix: use comments

This commit is contained in:
Johannes Frohnmeyer 2022-11-02 23:01:01 +01:00
parent 4710d20b7d
commit 4bc7ba11bf
Signed by: Johannes
GPG Key ID: E76429612C2929F4
10 changed files with 80 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
build.gradle.kts
View File

@ -1,5 +1,5 @@
plugins {
java
`java-library`
application
id("com.github.johnrengelman.shadow") version "7.1.2"
}
@ -13,16 +13,23 @@ application {
repositories {
mavenCentral()
maven("https://gitlab.com/api/v4/projects/35745143/packages/maven")
maven("https://maven.frohnmeyer-wds.de/artifacts")
}
dependencies {
implementation("net.freeutils:jlhttp:2.6")
implementation("com.amihaiemil.web:eo-yaml:6.1.0")
implementation("io.gitlab.jfronny:commons:2022.9.26+17-56-16")
implementation("io.gitlab.jfronny:commons-gson:2022.9.26+17-56-16")
implementation("io.gitlab.jfronny:commons:2022.11.2+21-36-56")
implementation("io.gitlab.jfronny:commons-gson:2022.11.2+21-36-56")
api("io.gitlab.jfronny.gson:gson-compile-core:1.0-SNAPSHOT")
compileOnly("io.gitlab.jfronny.gson:gson-compile-annotations:1.0-SNAPSHOT")
annotationProcessor("io.gitlab.jfronny.gson:gson-compile-processor:1.0-SNAPSHOT")
}
tasks.runShadow {
args("8002")
}
tasks.withType<JavaCompile> {
options.compilerArgs.add("-AgsonCompileNoReflect")
}

13
src/main/java/io/gitlab/jfronny/woodpecker/include/Main.java
View File

@ -1,12 +1,8 @@
package io.gitlab.jfronny.woodpecker.include;
import com.amihaiemil.eoyaml.*;
import io.gitlab.jfronny.commons.HttpUtils;
import io.gitlab.jfronny.commons.log.Logger;
import io.gitlab.jfronny.commons.serialize.Serializer;
import io.gitlab.jfronny.commons.serialize.gson.api.v1.GsonHolders;
import io.gitlab.jfronny.woodpecker.include.model.*;
import io.gitlab.jfronny.woodpecker.include.util.YamlTypeAdapter;
import net.freeutils.httpserver.HTTPServer;
import java.io.*;
@ -22,8 +18,6 @@ public class Main {
}
HTTPServer server = new HTTPServer(Integer.parseInt(args[0]));
HTTPServer.VirtualHost host = server.getVirtualHost(null);
GsonHolders.registerSerializer();
GsonHolders.registerTypeAdapter(YamlMapping.class, new YamlTypeAdapter());
HttpUtils.setUserAgent("Woodpecker-Include/1.0");
host.addContext("/ciconfig", (req, resp) -> {
Ref ref = new Ref();
@ -45,9 +39,8 @@ public class Main {
private static int processRequest(HTTPServer.Request req, HTTPServer.Response resp, Ref ref) throws IOException {
RequestModel request;
try (var is = req.getBody()) {
String body = new String(is.readAllBytes());
request = Serializer.getInstance().deserialize(body, RequestModel.class);
try (var isr = new InputStreamReader(req.getBody())) {
request = GC_RequestModel.read(isr);
}
ref.name = request.repo.full_name;
AtomicBoolean changed = new AtomicBoolean(false);
@ -55,7 +48,7 @@ public class Main {
if (changed.get()) {
resp.sendHeaders(200);
try (OutputStreamWriter writer = new OutputStreamWriter(resp.getBody())) {
Serializer.getInstance().serialize(response, writer);
GC_ResponseModel.write(writer, response);
}
LOG.info("Processed includes for pipeline for " + ref.name);
} else {

84
src/main/java/io/gitlab/jfronny/woodpecker/include/PipelineUnpacker.java
View File

@ -1,9 +1,8 @@
package io.gitlab.jfronny.woodpecker.include;
import com.amihaiemil.eoyaml.*;
import io.gitlab.jfronny.commons.HttpUtils;
import io.gitlab.jfronny.commons.StreamIterable;
import io.gitlab.jfronny.woodpecker.include.model.Pipeline;
import io.gitlab.jfronny.woodpecker.include.util.FilteringYamlMapping;
import java.io.*;
import java.net.URI;
@ -11,12 +10,17 @@ import java.net.URISyntaxException;
import java.nio.file.Paths;
import java.util.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
final class PipelineUnpacker implements BiConsumer<Pipeline, Consumer<Pipeline>> {
private static final String URL = "https?://(?:www\\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b[-a-zA-Z0-9()@:%_+.~#?&/=]*";
private static final Pattern INCLUDE = Pattern.compile("^#include (" + URL + ")$");
private static final Pattern LINK = Pattern.compile("^#link (" + URL + ")$");
private final AtomicBoolean changed;
private final Set<String> included = new HashSet<>();
private final Set<String> linked = new HashSet<>();
public PipelineUnpacker(AtomicBoolean changed) {
this.changed = changed;
@ -25,7 +29,7 @@ final class PipelineUnpacker implements BiConsumer<Pipeline, Consumer<Pipeline>>
@Override
public void accept(Pipeline pipeline, Consumer<Pipeline> pipelineConsumer) {
try {
processPipeline(pipeline, pipelineConsumer);
processPipeline(pipeline, pipelineConsumer, 0);
} catch (URISyntaxException e) {
throw new UncheckedIOException(new IOException("Could not find URL", e));
} catch (IOException e) {
@ -33,47 +37,59 @@ final class PipelineUnpacker implements BiConsumer<Pipeline, Consumer<Pipeline>>
}
}
private void processPipeline(Pipeline pipeline, Consumer<Pipeline> pipelineConsumer) throws URISyntaxException, IOException {
YamlNode include = pipeline.data.value("include");
if (include == null) {
private void processPipeline(Pipeline pipeline, Consumer<Pipeline> pipelineConsumer, int depth) throws URISyntaxException, IOException {
if (depth > 5) throw new IOException("Too many nested includes, a maximum of 5 is supported");
List<String> toLink = pipeline.data().lines()
.map(LINK::matcher)
.filter(Matcher::matches)
.map(s -> s.group(1))
.filter(s -> !linked.contains(s))
.toList();
boolean hasIncludes = pipeline.data().lines().anyMatch(INCLUDE.asPredicate());
if (toLink.isEmpty() && !hasIncludes) {
// Has no includes
pipelineConsumer.accept(pipeline);
return;
}
changed.set(true);
// Send included
//TODO switch back to switch pattern matching once that is available
if (include instanceof Scalar str) download(str.value(), null, pipelineConsumer);
else if (include instanceof YamlMapping map) {
for (YamlNode key : map.keys()) {
download(map.string(key), key.asScalar().value(), pipelineConsumer);
// Fill in includes and reprocess
if (hasIncludes) {
StringBuilder newData = new StringBuilder();
for (String line : new StreamIterable<>(pipeline.data().lines())) {
Matcher matcher = INCLUDE.matcher(line);
if (!matcher.matches()) newData.append(line);
else {
newData.append(download(matcher.group(1)).data());
}
newData.append('\n');
}
} else if (include instanceof YamlSequence seq) {
for (YamlNode node : seq) {
download(node.asScalar().value(), null, pipelineConsumer);
}
} else throw new IllegalStateException("Unexpected value: " + include);
if (pipeline.data.keys().size() > 1) {
processPipeline(new Pipeline(pipeline.name(), newData.toString()), pipelineConsumer, depth + 1);
return;
}
// Link additional pipelines
for (String url : toLink) {
if (linked.contains(url)) continue;
linked.add(url);
processPipeline(download(url), pipelineConsumer, depth + 1);
}
if (pipeline.data().lines()
.anyMatch(INCLUDE.asPredicate().negate()
.and(LINK.asPredicate().negate())
.and(Predicate.not(String::isBlank)))) {
// More than just includes: generate override without include node
Pipeline filtered = new Pipeline();
filtered.name = pipeline.name;
pipeline.data = new FilteringYamlMapping(pipeline.data, s -> !s.asScalar().value().equals("include"));
pipelineConsumer.accept(filtered);
pipelineConsumer.accept(pipeline);
}
}
private void download(String url, String fileName, Consumer<Pipeline> pipelineConsumer) throws URISyntaxException, IOException {
if (included.contains(url)) return;
included.add(url);
private Pipeline download(String url) throws URISyntaxException, IOException {
URI uri = new URI(url.replace(" ", "%20"));
if (!"http".equalsIgnoreCase(uri.getScheme()) && !"https".equalsIgnoreCase(uri.getScheme())) {
throw new URISyntaxException(url, "Could not find scheme");
}
try (InputStream is = HttpUtils.get(url).sendInputStream()) {
Pipeline pipeline = new Pipeline();
pipeline.name = fileName != null ? fileName : Paths.get(new URI(url.replace(" ", "%20")).getPath()).getFileName().toString();
pipeline.data = Yaml.createYamlInput(is).readYamlMapping();
processPipeline(pipeline, pipelineConsumer);
}
String fileName = Paths.get(new URI(url.replace(" ", "%20")).getPath()).getFileName().toString();
return new Pipeline(fileName, HttpUtils.get(url).sendString());
}
}

3
src/main/java/io/gitlab/jfronny/woodpecker/include/model/Build.java
View File

@ -1,7 +1,10 @@
package io.gitlab.jfronny.woodpecker.include.model;
import io.gitlab.jfronny.gson.compile.annotations.GSerializable;
import java.util.List;
@GSerializable
public class Build {
public Long id;
public Long number;

20
src/main/java/io/gitlab/jfronny/woodpecker/include/model/Pipeline.java
View File

@ -1,21 +1,7 @@
package io.gitlab.jfronny.woodpecker.include.model;
import com.amihaiemil.eoyaml.YamlMapping;
import io.gitlab.jfronny.gson.compile.annotations.GSerializable;
public class Pipeline {
public String name;
public YamlMapping data;
@Override
public String toString() {
return "{\"name\":\"" + escape(name) + ",\"data\":\"" + escape(data.toString()) + '}';
}
private static String escape(String text) {
return text
.replace("\\", "\\\\")
.replace("\"", "\\\"")
.replace("\r", "\\r")
.replace("\n", "\\n");
}
@GSerializable
public record Pipeline(String name, String data) {
}

2
src/main/java/io/gitlab/jfronny/woodpecker/include/model/Repo.java
View File

@ -1,9 +1,11 @@
package io.gitlab.jfronny.woodpecker.include.model;
import io.gitlab.jfronny.gson.annotations.SerializedName;
import io.gitlab.jfronny.gson.compile.annotations.GSerializable;
import java.util.List;
@GSerializable
public class Repo {
public Long id;
public String owner;

3
src/main/java/io/gitlab/jfronny/woodpecker/include/model/RequestModel.java
View File

@ -1,7 +1,10 @@
package io.gitlab.jfronny.woodpecker.include.model;
import io.gitlab.jfronny.gson.compile.annotations.GSerializable;
import java.util.List;
@GSerializable
public class RequestModel {
public Repo repo;
public Build pipeline;

26
src/main/java/io/gitlab/jfronny/woodpecker/include/model/ResponseModel.java
View File

@ -1,27 +1,9 @@
package io.gitlab.jfronny.woodpecker.include.model;
import io.gitlab.jfronny.gson.compile.annotations.GSerializable;
import java.util.List;
public class ResponseModel {
public List<Pipeline> configs;
public ResponseModel() {
}
public ResponseModel(List<Pipeline> configs) {
this.configs = List.copyOf(configs);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("{\"configs\":[");
boolean first = false;
for (Pipeline config : configs) {
if (!first) sb.append(',');
first = true;
sb.append(config.toString());
}
sb.append("]}");
return sb.toString();
}
@GSerializable
public record ResponseModel(List<Pipeline> configs) {
}

32
src/main/java/io/gitlab/jfronny/woodpecker/include/util/FilteringYamlMapping.java
View File

@ -1,32 +0,0 @@
package io.gitlab.jfronny.woodpecker.include.util;
import com.amihaiemil.eoyaml.*;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;
public class FilteringYamlMapping extends BaseYamlMapping {
private final YamlMapping base;
private final Predicate<YamlNode> filter;
public FilteringYamlMapping(YamlMapping base, Predicate<YamlNode> filter) {
this.base = base;
this.filter = filter;
}
@Override
public Set<YamlNode> keys() {
return base.keys().stream().filter(filter).collect(Collectors.toSet());
}
@Override
public YamlNode value(YamlNode key) {
return filter.test(key) ? base.value(key) : null;
}
@Override
public Comment comment() {
return base.comment();
}
}

21
src/main/java/io/gitlab/jfronny/woodpecker/include/util/YamlTypeAdapter.java
View File

@ -1,21 +0,0 @@
package io.gitlab.jfronny.woodpecker.include.util;
import com.amihaiemil.eoyaml.Yaml;
import com.amihaiemil.eoyaml.YamlMapping;
import io.gitlab.jfronny.gson.TypeAdapter;
import io.gitlab.jfronny.gson.stream.JsonReader;
import io.gitlab.jfronny.gson.stream.JsonWriter;
import java.io.IOException;
public class YamlTypeAdapter extends TypeAdapter<YamlMapping> {
@Override
public void write(JsonWriter jsonWriter, YamlMapping yamlMapping) throws IOException {
jsonWriter.value(yamlMapping.toString());
}
@Override
public YamlMapping read(JsonReader jsonReader) throws IOException {
return Yaml.createYamlInput(jsonReader.nextString()).readYamlMapping();
}
}