eclipse-sailing-analytics/sailing-analytics
1
0
Fork 0
mirror of https://github.com/eclipse-sailing-analytics/sailing-analytics.git synced 2026-09-17 11:19:15 +00:00
Code Activity

bug6173: interims backup commit; working towards reducing number of requests

Browse Source
This commit is contained in:
Axel Uhl
2025-11-05 18:48:33 +01:00
parent 50fa9267de
commit 1eab2fca9c
3 changed files with 126 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/com.sap.sse.landscape/src/com/sap/sse/landscape/impl/AbstractReleaseRepository.java
+2 -11
View File
@@ -1,7 +1,5 @@
package com.sap.sse.landscape.impl;
import java.util.Iterator;
import com.sap.sse.common.Util;
import com.sap.sse.landscape.Release;
import com.sap.sse.landscape.ReleaseRepository;
@@ -19,17 +17,10 @@ public abstract class AbstractReleaseRepository implements ReleaseRepository {
return defaultReleaseNamePrefix;
}
protected abstract Iterable<Release> getAvailableReleases();
@Override
public Iterator<Release> iterator() {
return getAvailableReleases().iterator();
}
@Override
public Release getLatestRelease(String releaseNamePrefix) {
Release result = null;
for (final Release release : getAvailableReleases()) {
for (final Release release : this) { // invokes the iterator() method
if (release.getBaseName().equals(releaseNamePrefix) &&
(result == null || release.getCreationDate().after(result.getCreationDate()))) {
result = release;
@@ -40,6 +31,6 @@ public abstract class AbstractReleaseRepository implements ReleaseRepository {
@Override
public Release getRelease(String releaseName) {
return Util.first(Util.filter(getAvailableReleases(), r->r.getName().equals(releaseName)));
return Util.first(Util.filter(this, r->r.getName().equals(releaseName)));
}
}
java/com.sap.sse.landscape/src/com/sap/sse/landscape/impl/FolderBasedReleaseRepositoryImpl.java
+7 -1
View File
@@ -6,6 +6,7 @@ import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.logging.Level;
@@ -39,7 +40,12 @@ public class FolderBasedReleaseRepositoryImpl extends AbstractReleaseRepository
return repositoryBase;
}
protected Iterable<Release> getAvailableReleases() {
@Override
public Iterator<Release> iterator() {
return getAvailableReleases().iterator();
}
private Iterable<Release> getAvailableReleases() {
final List<Release> result = new LinkedList<>();
try {
final URLConnection connection = HttpUrlConnectionHelper.redirectConnection(new URL(getRepositoryBase()));
java/com.sap.sse.landscape/src/com/sap/sse/landscape/impl/GithubReleasesRepository.java
+117 -16
View File
@@ -3,10 +3,14 @@ package com.sap.sse.landscape.impl;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.text.SimpleDateFormat;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.TreeMap;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -16,27 +20,35 @@ import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import com.sap.sse.common.TimePoint;
import com.sap.sse.common.Util.Pair;
import com.sap.sse.landscape.Release;
import com.sap.sse.landscape.ReleaseRepository;
import com.sap.sse.util.HttpUrlConnectionHelper;
/**
* Assumes a public GitHub repository where releases can be freely downloaded from
* <code>https://github.com/{owner}/{repo}/releases/download/{release-name}</code>.
* <code>https://github.com/{owner}/{repo}/releases/download/{release-name}</code>. The GitHub
* {@code /releases} end point delivers the releases in descending chronological order, so
* newest releases first. With this, we can cache old results and try to get along with the
* harsh rate limit of only 60 requests per hour when used without authentication.
*
* @author Axel Uhl (d043530)
*/
public class GithubReleasesRepository extends AbstractReleaseRepository implements ReleaseRepository {
private final static Logger logger = Logger.getLogger(GithubReleasesRepository.class.getName());
private static final SimpleDateFormat isoDateTimeFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssX");
private final static String GITHUB_API_BASE_URL = "https://api.github.com";
private final static String GITHUB_BASE_URL = "https://github.com";
private final String owner;
private final String repositoryName;
private final TreeMap<TimePoint, Release> releasesByPublishingTimePoint;
public GithubReleasesRepository(String owner, String repositoryName, String defaultReleaseNamePrefix) {
super(defaultReleaseNamePrefix);
this.owner = owner;
this.repositoryName = repositoryName;
this.releasesByPublishingTimePoint = new TreeMap<>();
}
private String getRepositoryPath() {
@@ -44,7 +56,7 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
}
private String getReleasesURL() {
return GITHUB_API_BASE_URL+"/repos/"+getRepositoryPath()+"/releases?per_page=100"; // TODO for unauthenticated requests there is a harsh rate limit of 60 requests per hour...
return GITHUB_API_BASE_URL+"/repos/"+getRepositoryPath()+"/releases?per_page=100";
}
@Override
@@ -52,9 +64,84 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
return new GithubRelease(releaseName, GITHUB_BASE_URL+"/"+getRepositoryPath()+"/releases/download/"+releaseName+"/"+releaseName+Release.ARCHIVE_EXTENSION,
GITHUB_BASE_URL+"/"+getRepositoryPath()+"/releases/download/"+releaseName+"/"+Release.RELEASE_NOTES_FILE_NAME);
}
/**
* Always fetches the first page from the {@code /releases} end point and starts constructing releases, until a
* publishing time point overlap with {@link GithubReleasesRepository#releasesByPublishingTimePoint} is found. Then
* we know we can continue to enumerate the remaining releases from that cache.
* <p>
*
* All releases found by loading a page are added to the
* {@link GithubReleasesRepository#releasesByPublishingTimePoint} cache.
*
* @author Axel Uhl (d043530)
*
*/
private class ReleaseIterator implements Iterator<Release> {
private String nextPageURL;
private Iterator<Pair<TimePoint, GithubRelease>> publishingTimePointsAndReleasesFromCurrentPageIterator;
private ReleaseIterator() throws MalformedURLException, IOException, ParseException {
nextPageURL = getReleasesURL();
loadNextPage();
}
private void loadNextPage() throws MalformedURLException, IOException, ParseException {
final List<Pair<TimePoint, GithubRelease>> result = new LinkedList<>();
final URLConnection connection = HttpUrlConnectionHelper.redirectConnection(new URL(nextPageURL));
final InputStream index = (InputStream) connection.getContent();
final String linkHeader = connection.getHeaderField("link");
nextPageURL = getNextPageURL(linkHeader);
final JSONArray releasesJson = (JSONArray) new JSONParser().parse(new InputStreamReader(index));
for (final Object releaseObject : releasesJson) {
final Pair<TimePoint, GithubRelease> publishedAtAndRelease = getPublishedAtAndReleaseFromJson((JSONObject) releaseObject);
releasesByPublishingTimePoint.put(publishedAtAndRelease.getA(), publishedAtAndRelease.getB());
result.add(publishedAtAndRelease);
}
publishingTimePointsAndReleasesFromCurrentPageIterator = result.iterator();
}
@Override
public boolean hasNext() {
return publishingTimePointsAndReleasesFromCurrentPageIterator.hasNext() || nextPageURL != null;
}
@Override
public Release next() {
if (!publishingTimePointsAndReleasesFromCurrentPageIterator.hasNext()) {
try {
// FIXME bug6173: only load next page if we have to... we may already have created an overlap with the cache from releasesByPublishingTimePoint
loadNextPage();
} catch (IOException | ParseException e) {
throw new RuntimeException(e);
}
}
return publishingTimePointsAndReleasesFromCurrentPageIterator.next().getB();
}
}
@Override
public Iterator<Release> iterator() {
try {
return new ReleaseIterator();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@Override
protected Iterable<Release> getAvailableReleases() {
public Release getLatestRelease(String releaseNamePrefix) {
// TODO Auto-generated method stub
return super.getLatestRelease(releaseNamePrefix);
}
/**
* Enumerating all releases of the GitHub repo is possible but goes against the harsh rate limit when used without
* an access token (currently only 60 requests per hour), so should ideally be avoided altogether. And if it is ever called,
* we will cache the results, so that for later requests we typically need to query only a single page, delivering the latest
* additions, if any.
*/
private Iterable<Release> getAvailableReleases() {
final List<Release> result = new LinkedList<>();
try {
String nextPageURL = getReleasesURL();
@@ -74,21 +161,35 @@ public class GithubReleasesRepository extends AbstractReleaseRepository implemen
private void addAllReleasesTo(JSONArray releasesJson, List<Release> result) {
for (final Object releaseObject : releasesJson) {
final JSONObject releaseJson = (JSONObject) releaseObject;
final String name = releaseJson.get("name").toString();
String archiveDownloadURL = null;
String releaseNotesURL = null;
for (final Object archiveAsset : (JSONArray) releaseJson.get("assets")) {
final JSONObject archiveAssetJson = (JSONObject) archiveAsset;
if (archiveAssetJson.get("content_type").equals("application/x-tar")) {
archiveDownloadURL = archiveAssetJson.get("browser_download_url").toString();
} else if (archiveAssetJson.get("name").equals(Release.RELEASE_NOTES_FILE_NAME)) {
releaseNotesURL = archiveAssetJson.get("browser_download_url").toString();
}
}
result.add(new GithubRelease(name, archiveDownloadURL, releaseNotesURL));
final Pair<TimePoint, GithubRelease> publishedAtAndRelease = getPublishedAtAndReleaseFromJson((JSONObject) releaseObject);
result.add(publishedAtAndRelease.getB());
releasesByPublishingTimePoint.put(publishedAtAndRelease.getA(), publishedAtAndRelease.getB());
}
}
private Pair<TimePoint, GithubRelease> getPublishedAtAndReleaseFromJson(JSONObject releaseJson) {
final String name = releaseJson.get("name").toString();
final String publishedAtISO = releaseJson.get("published_at").toString();
TimePoint publishedAt;
try {
publishedAt = TimePoint.of(isoDateTimeFormat.parse(publishedAtISO));
} catch (java.text.ParseException e) {
logger.warning("Couldn't read published_at time stamp for release "+name+": "+publishedAtISO);
throw new RuntimeException(e);
}
String archiveDownloadURL = null;
String releaseNotesURL = null;
for (final Object archiveAsset : (JSONArray) releaseJson.get("assets")) {
final JSONObject archiveAssetJson = (JSONObject) archiveAsset;
if (archiveAssetJson.get("content_type").equals("application/x-tar")) {
archiveDownloadURL = archiveAssetJson.get("browser_download_url").toString();
} else if (archiveAssetJson.get("name").equals(Release.RELEASE_NOTES_FILE_NAME)) {
releaseNotesURL = archiveAssetJson.get("browser_download_url").toString();
}
}
final GithubRelease release = new GithubRelease(name, archiveDownloadURL, releaseNotesURL);
return new Pair<>(publishedAt, release);
}
private static final Pattern nextPagePattern = Pattern.compile(".*<([^<]*)>; rel=\"next\".*");
String getNextPageURL(String linkHeader) {