package io.gitlab.jfronny.woodpecker.include; import io.gitlab.jfronny.commons.HttpUtils; import io.gitlab.jfronny.commons.StreamIterable; import io.gitlab.jfronny.woodpecker.include.model.Pipeline; import java.io.*; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Paths; import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.*; import java.util.regex.Matcher; import java.util.regex.Pattern; final class PipelineUnpacker implements BiConsumer> { private static final String URL = "https?://(?:www\\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b[-a-zA-Z0-9()@:%_+.~#?&/=]*"; private static final Pattern INCLUDE = Pattern.compile("^#include (" + URL + ")$"); private static final Pattern LINK = Pattern.compile("^#link (" + URL + ")$"); private final AtomicBoolean changed; private final Set linked = new HashSet<>(); public PipelineUnpacker(AtomicBoolean changed) { this.changed = changed; } @Override public void accept(Pipeline pipeline, Consumer pipelineConsumer) { try { processPipeline(pipeline, pipelineConsumer, 0); } catch (URISyntaxException e) { throw new UncheckedIOException(new IOException("Could not find URL", e)); } catch (IOException e) { throw new UncheckedIOException(e); } } private void processPipeline(Pipeline pipeline, Consumer pipelineConsumer, int depth) throws URISyntaxException, IOException { if (depth > 5) throw new IOException("Too many nested includes, a maximum of 5 is supported"); List toLink = pipeline.data().lines() .map(LINK::matcher) .filter(Matcher::matches) .map(s -> s.group(1)) .filter(s -> !linked.contains(s)) .toList(); boolean hasIncludes = pipeline.data().lines().anyMatch(INCLUDE.asPredicate()); if (toLink.isEmpty() && !hasIncludes) { // Has no includes pipelineConsumer.accept(pipeline); return; } changed.set(true); // Fill in includes and reprocess if (hasIncludes) { StringBuilder newData = new StringBuilder(); for (String line : new StreamIterable<>(pipeline.data().lines())) { Matcher matcher = INCLUDE.matcher(line); if (!matcher.matches()) newData.append(line); else newData.append(download(matcher.group(1)).data()); newData.append('\n'); } processPipeline(new Pipeline(pipeline.name(), newData.toString()), pipelineConsumer, depth + 1); return; } // Link additional pipelines for (String url : toLink) { if (linked.contains(url)) continue; linked.add(url); processPipeline(download(url), pipelineConsumer, depth + 1); } // Filter StringBuilder newDate = new StringBuilder(); boolean foundContent = false; for (String line : new StreamIterable<>(pipeline.data().lines())) { if (LINK.matcher(line).matches()) continue; if (!line.isBlank()) foundContent = true; newDate.append(line).append('\n'); } if (foundContent) { pipelineConsumer.accept(new Pipeline(pipeline.name(), newDate.toString().trim())); } } private Pipeline download(String url) throws URISyntaxException, IOException { URI uri = new URI(url.replace(" ", "%20")); if (!"http".equalsIgnoreCase(uri.getScheme()) && !"https".equalsIgnoreCase(uri.getScheme())) { throw new URISyntaxException(url, "Could not find scheme"); } String fileName = Paths.get(new URI(url.replace(" ", "%20")).getPath()).getFileName().toString(); return new Pipeline(fileName, HttpUtils.get(url).sendString()); } }