[Releng] Improve the Temurim indexer to use github APIs to find releases
diff --git a/plugins/org.eclipse.justj.codegen/cache/https/api.github.com/repos/.gitignore b/plugins/org.eclipse.justj.codegen/cache/https/api.github.com/repos/.gitignore
new file mode 100644
index 0000000..330eff0
--- /dev/null
+++ b/plugins/org.eclipse.justj.codegen/cache/https/api.github.com/repos/.gitignore
@@ -0,0 +1 @@
+/adoptium/
diff --git a/plugins/org.eclipse.justj.codegen/src/org/eclipse/justj/codegen/model/util/TemurinIndexer.java b/plugins/org.eclipse.justj.codegen/src/org/eclipse/justj/codegen/model/util/TemurinIndexer.java
index c291a70..79ebc4b 100644
--- a/plugins/org.eclipse.justj.codegen/src/org/eclipse/justj/codegen/model/util/TemurinIndexer.java
+++ b/plugins/org.eclipse.justj.codegen/src/org/eclipse/justj/codegen/model/util/TemurinIndexer.java
@@ -11,44 +11,43 @@
package org.eclipse.justj.codegen.model.util;
-import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.io.InputStream;
-import java.io.InputStreamReader;
import java.io.PrintStream;
-import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
-import java.util.Comparator;
import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import java.util.stream.Collectors;
+
+import org.eclipse.emf.common.util.URI;
+import org.eclipse.emf.ecore.resource.URIConverter;
public class TemurinIndexer
{
-
- private static final Pattern TAG_NAME_ANY = Pattern.compile("\"name\"\\s*:\\s*\"([^\"]+)\"");
-
- private static final Pattern TAG_NAME = Pattern.compile("\"name\"\\s*:\\s*\"(jdk-([0-9]+)(?:\\.([0-9]+))?(?:\\.([0-9]+))?(?:\\.([0-9]+))?\\+([0-9]+))\"");
-
- private static final Pattern BETA_TAG_NAME = Pattern.compile("\"name\"\\s*:\\s*\"(jdk([0-9]+)-(20[0-9]+)-([0-9]+)-([0-9]+)-([0-9]+)-([0-9]+)-beta)\"");
-
private static final Pattern JDK_URL = Pattern.compile(//
- "href=\"/(adoptium/temurin[0-9]+-binaries/releases/download/jdk[0-9]*-[^/]+/OpenJDK[^-]*-jdk_[^\"]*\\.(zip|tar.gz))\"");
+ "\"browser_download_url\":\"https://github.com/(adoptium/temurin[0-9]+-binaries/releases/download/jdk[0-9]*-[^/]+/OpenJDK[^-]*-jdk_[^\"]*\\.(zip|tar.gz))\"");
+
+ private static final Pattern ASSETS_PATTERN = Pattern.compile("\"assets\":\\[([^]]+)\\]");
private static final String BEGIN_MARKER = "// Begin generated by /org.eclipse.justj.codegen/src/org/eclipse/justj/codegen/model/util/TemurinIndexer.java";
private static final String END_MARKER = "// End generated by /org.eclipse.justj.codegen/src/org/eclipse/justj/codegen/model/util/TemurinIndexer.java";
+ private static Path cache;
+
public static void main(String[] args) throws Exception
{
+ if (args.length > 1)
+ {
+ cache = Path.of(args[0]);
+ }
+
PrintStream out = System.out;
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()
@@ -68,69 +67,21 @@
out.println(BEGIN_MARKER);
- for (String repo : new String []{ "11", "17", "18", "19" })
+ for (String repo : new String []{ "11", "17", "18", "19", "20" })
{
- List<String> lines = getTags(repo);
- Map<Long, String> versionTags = new TreeMap<>(Comparator.reverseOrder());
- for (String line : lines)
- {
- for (Matcher matcher = TAG_NAME.matcher(line); matcher.find();)
- {
- long version = Integer.parseInt(matcher.group(2));
- version = 1000L * version + (matcher.group(3) == null ? 0 : Integer.parseInt(matcher.group(3)));
- version = 100L * version + (matcher.group(4) == null ? 0 : Integer.parseInt(matcher.group(4)));
- version = 100L * version + (matcher.group(5) == null ? 0 : Integer.parseInt(matcher.group(5)));
- version = 100L * version + Integer.parseInt(matcher.group(6));
- versionTags.put(version, matcher.group(1));
- }
- }
+ URI releasesURI = URI.createURI("https://api.github.com/repos/adoptium/temurin" + repo + "-binaries/releases");
+ String content = getContent(releasesURI);
+ Matcher matcher2 = ASSETS_PATTERN.matcher(content);
+ matcher2.find();
+ String assets = matcher2.group(1);
- if (versionTags.isEmpty())
+ List<String> jdkDownloadURLs = new ArrayList<>();
+ for (Matcher matcher3 = JDK_URL.matcher(assets); matcher3.find();)
{
- for (String line : lines)
- {
- for (Matcher matcher = BETA_TAG_NAME.matcher(line); matcher.find();)
- {
- long version = Integer.parseInt(matcher.group(2));
- version = 100L * version + Integer.parseInt(matcher.group(3));
- version = 100L * version + Integer.parseInt(matcher.group(4));
- version = 100L * version + Integer.parseInt(matcher.group(5));
- version = 1000L * version + Integer.parseInt(matcher.group(6));
- version = 1000L * version + Integer.parseInt(matcher.group(7));
- versionTags.put(version, matcher.group(1));
- }
- }
-
- if (versionTags.isEmpty())
- {
- for (String line : lines)
- {
- for (Matcher matcher = TAG_NAME_ANY.matcher(line); matcher.find();)
- {
- String name = matcher.group(1);
- System.err.println("###" + name);
- }
- }
- }
+ jdkDownloadURLs.add("https://github.com/" + matcher3.group(1).replace("%2B", "+"));
}
out.println();
- out.println("// Tags " + versionTags.values().stream().limit(10).collect(Collectors.joining(", ")));
-
- String latest = versionTags.entrySet().iterator().next().getValue();
- URL downloadsURL = new URL("https://github.com/adoptium/temurin" + repo + "-binaries/releases/tag/" + latest);
- List<String> jdkDownloadURLs = new ArrayList<>();
- try (InputStream download = downloadsURL.openStream())
- {
- List<String> downloads = new BufferedReader(new InputStreamReader(download)).lines().collect(Collectors.toList());
- for (String downloadLine : downloads)
- {
- for (Matcher matcher = JDK_URL.matcher(downloadLine); matcher.find();)
- {
- jdkDownloadURLs.add("https://github.com/" + matcher.group(1).replace("%2B", "+"));
- }
- }
- }
out.println("def java" + repo + "Adoptium = [");
out.println(" label: 'Java " + repo + " (Adoptium)',");
@@ -195,9 +146,9 @@
out.println();
out.println(END_MARKER);
- if (args.length == 1)
+ if (args.length > 0)
{
- Path target = Paths.get(args[0]);
+ Path target = Paths.get(args[args.length - 1]);
String contents = Files.readString(target);
String newGeneratedContents = new String(byteArrayOutputStream.toByteArray(), StandardCharsets.UTF_8);
contents = contents.replaceAll("(?s)" + Pattern.quote(BEGIN_MARKER) + ".*" + Pattern.quote(END_MARKER) + "\r?\n", Matcher.quoteReplacement(newGeneratedContents));
@@ -205,28 +156,6 @@
}
}
- private static List<String> getTags(String repo) throws Exception
- {
- List<String> result = new ArrayList<>();
- LOOP: for (int i = 1; i < 10; ++i)
- {
- URL url = new URL("https://api.github.com/repos/adoptium/temurin" + repo + "-binaries/tags?page=" + i);
- try (InputStream input = url.openStream())
- {
- List<String> lines = new BufferedReader(new InputStreamReader(input)).lines().collect(Collectors.toList());
- for (String line : lines)
- {
- if (result.contains(line))
- {
- break LOOP;
- }
- result.add(line);
- }
- }
- }
- return result;
- }
-
private static String getURL(List<String> downloadURLs, String type)
{
for (String downloadURL : downloadURLs)
@@ -239,4 +168,41 @@
return null;
}
+
+ private static Path getCachePath(URI uri)
+ {
+ if (cache == null)
+ {
+ return null;
+ }
+
+ String decodedURI = URI.decode(uri.toString());
+ String[] uriSegments = decodedURI.split("[:/?#&;]+");
+ Path result = cache.resolve(String.join("/", uriSegments));
+ if (uri.hasTrailingPathSeparator())
+ {
+ return result.resolve("-folder-contents");
+ }
+ return result;
+ }
+
+ private static String getContent(URI uri) throws IOException
+ {
+ Path path = getCachePath(uri);
+ if (path != null && Files.isRegularFile(path))
+ {
+ return Files.readString(path);
+ }
+
+ try (InputStream in = URIConverter.INSTANCE.createInputStream(uri))
+ {
+ String content = new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ if (path != null)
+ {
+ Files.createDirectories(path.getParent());
+ Files.writeString(path, content);
+ }
+ return content;
+ }
+ }
}
