Merge branch 'release/0.0.15'

README.md

@@ -9,15 +9,15 @@
 Export your favorite GitHub repositories to Prometheus
 
 * Use it _as a service_: See https://gh.skuzzle.de for instructions
-* Deploy it _on-premise_: `docker pull ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.14`
+* Deploy it _on-premise_: `docker pull ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.15`
 
 ## On-Premise deployment with docker
 This application can easily be run as a docker container in whatever environment you like:
 
 ```
 docker run -p 8080:8080 \
     -e WEB_ALLOWANONYMOUSSCRAPE=true \
-    ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.14
+    ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.15
 ```
 
 With _anonymous scraping_ allowed, you can now easily view the scrape results directly in the browser by navigating to
@@ -38,7 +38,7 @@ scrape_configs:
 In case you want to enforce authenticated scrapes only, use this configuration instead:
 ```
 docker run -p 8080:8080 \
-    ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.14
+    ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.15
 ```
 
 Scraping now requires a GitHub access token, otherwise the service will respond with 401/Unauthorized. 

RELEASE_NOTES.md

@@ -2,8 +2,8 @@
 
 [![Coverage Status](https://coveralls.io/repos/github/skuzzle/gh-prom-exporter/badge.svg?branch=master)](https://coveralls.io/github/skuzzle/gh-prom-exporter?branch=master) [![Twitter Follow](https://img.shields.io/twitter/follow/skuzzleOSS.svg?style=social)](https://twitter.com/skuzzleOSS)
 
-* Upgrade to github-api 1.308 (coming from 1.135)
+* Add option to pass GitHub token via query parameter: `?token=...` (Note: this is primarily intended for local testing and should not be used in production)
 
 ```
-docker pull ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.14
+docker pull ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.15
 ```

pom.xml

@@ -11,7 +11,7 @@
 
     <groupId>de.skuzzle.ghpromexporter</groupId>
     <artifactId>gh-prom-exporter</artifactId>
-    <version>0.0.14</version>
+    <version>0.0.15</version>
 
     <name>gh-prom-exporter</name>
     <description>Export GitHub repository metrics in prometheus format</description>

readme/RELEASE_NOTES.md

@@ -2,7 +2,7 @@
 
 [![Coverage Status](https://coveralls.io/repos/github/${github.user}/${github.name}/badge.svg?branch=${github.main-branch})](https://coveralls.io/github/${github.user}/${github.name}?branch=${github.main-branch}) [![Twitter Follow](https://img.shields.io/twitter/follow/skuzzleOSS.svg?style=social)](https://twitter.com/skuzzleOSS)
 
-* Upgrade to github-api 1.308 (coming from 1.135)
+* Add option to pass GitHub token via query parameter: `?token=...` (Note: this is primarily intended for local testing and should not be used in production)
 
 ```
 docker pull ${docker.image.name}:${project.version}

src/main/java/de/skuzzle/ghpromexporter/appmetrics/AppMetrics.java

@@ -18,7 +18,6 @@ public final class AppMetrics {
     private static final Counter scrapeFailures = Metrics.counter(NAMESPACE + "scrape_failures");
     private static final Counter abuses = Metrics.counter(NAMESPACE + "abuses");
     private static final Counter apiCalls = Metrics.counter(NAMESPACE + "api_calls");
-    private static final Counter rateLimitHits = Metrics.counter(NAMESPACE + "rate_limit_hits");
 
     public static Timer scrapeDuration() {
         return scrapeDuration;
@@ -39,8 +38,4 @@ public static Counter abuses() {
     public static Counter apiCalls() {
         return apiCalls;
     }
-
-    public static Counter rateLimitHits() {
-        return rateLimitHits;
-    }
 }

src/main/java/de/skuzzle/ghpromexporter/github/GitHubAuthentication.java

@@ -32,6 +32,11 @@ public static GitHubAuthentication anonymous(InetAddress source) {
 
     GitHub connectToGithub() throws IOException;
 
+    /**
+     * Remove this authentication from the internal cache.
+     */
+    void disconnect();
+
     boolean isAnonymous();
 
 }

src/main/java/de/skuzzle/ghpromexporter/github/GitHubFactory.java

@@ -1,13 +1,13 @@
 package de.skuzzle.ghpromexporter.github;
 
 import java.io.IOException;
-import java.net.HttpURLConnection;
-import java.net.URL;
 import java.util.concurrent.ExecutionException;
 
 import org.kohsuke.github.GitHub;
 import org.kohsuke.github.GitHubBuilder;
-import org.kohsuke.github.HttpConnector;
+import org.kohsuke.github.connector.GitHubConnector;
+import org.kohsuke.github.connector.GitHubConnectorRequest;
+import org.kohsuke.github.connector.GitHubConnectorResponse;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -39,16 +39,19 @@ private static GitHub buildGitHubFor(InternalGitHubAuthentication origin) throws
         log.info("Creating new GitHub connection for {}", origin);
         return origin.consumeBuilder(new GitHubBuilder())
                 .withConnector(new RequestCountingHttpConnector())
-                .withRateLimitHandler(new LoggingRateLimiter(origin))
                 .build();
     }
 
-    private static class RequestCountingHttpConnector implements HttpConnector {
+    static void disconnect(GitHubAuthentication authentication) {
+        CACHED_GITHUBS.invalidate(authentication);
+    }
+
+    private static class RequestCountingHttpConnector implements GitHubConnector {
 
         @Override
-        public HttpURLConnection connect(URL url) throws IOException {
+        public GitHubConnectorResponse send(GitHubConnectorRequest connectorRequest) throws IOException {
             AppMetrics.apiCalls().increment();
-            return DEFAULT.connect(url);
+            return DEFAULT.send(connectorRequest);
         }
 
     }

src/main/java/de/skuzzle/ghpromexporter/github/InternalGitHubAuthentication.java

@@ -15,6 +15,11 @@ default GitHub connectToGithub() throws IOException {
         return GitHubFactory.createGitHub(this);
     }
 
+    @Override
+    default void disconnect() {
+        GitHubFactory.disconnect(this);
+    }
+
     record TokenAuthentication(String token) implements InternalGitHubAuthentication {
 
         @Override

src/main/java/de/skuzzle/ghpromexporter/github/ScrapableRepository.java

@@ -35,6 +35,15 @@ public static ScrapableRepository load(GitHubAuthentication authentication, Stri
         try {
             final GitHub gitHub = authentication.connectToGithub();
             final GHRepository repository = gitHub.getRepository(repositoryFullName);
+
+            // GH Issue #8: https://github.com/skuzzle/gh-prom-exporter/issues/8
+            // Git Hub API Note:
+            // Statistics will be generated by GitHub asynchronously upon the first
+            // getStatistics call
+            // Until the statistics are available, getCodeFrequency and
+            // getContributorStats throw NPE, which might lead to false positives during
+            // abuse detection. To mitigate this, abuse detection must be either disabled
+            // or threshold must be set to a high value.
             final GHRepositoryStatistics statistics = repository.getStatistics();
             final List<CodeFrequency> codeFrequency = statistics.getCodeFrequency();
             final IntSummaryStatistics commitsToMainBranch = StreamSupport

src/main/java/de/skuzzle/ghpromexporter/web/AbuseLimiter.java

@@ -9,6 +9,7 @@
 import com.google.common.cache.Cache;
 
 import de.skuzzle.ghpromexporter.appmetrics.AppMetrics;
+import de.skuzzle.ghpromexporter.github.GitHubAuthentication;
 import reactor.core.publisher.Mono;
 
 class AbuseLimiter {
@@ -28,18 +29,22 @@ public AbuseLimiter(Cache<InetAddress, Integer> abusers, int abuseLimit) {
      * contain just an arbitrary object.
      *
      * @param origin The origin IP to check.
+     * @param authentication The resolved GitHubAuthentication object for the current
+     *            request.
      * @return An empty Mono if the abuse limit was violated by that IP.
      */
-    Mono<Object> blockAbusers(InetAddress origin) {
+    Mono<Object> blockAbusers(InetAddress origin, GitHubAuthentication authentication) {
         return Mono.fromSupplier(() -> _0IfNull(abusers.getIfPresent(origin)))
-                .filter(actualAbuses -> abuseLimitExceeded(origin, actualAbuses))
+                .filter(actualAbuses -> abuseLimitExceeded(origin, authentication, actualAbuses))
                 .map(abuses -> (Object) abuses);
     }
 
-    private boolean abuseLimitExceeded(InetAddress origin, int actualAbuses) {
+    private boolean abuseLimitExceeded(InetAddress origin, GitHubAuthentication authentication, int actualAbuses) {
         if (actualAbuses >= abuseLimit) {
             AppMetrics.abuses().increment();
-            log.warn("Abuse limit exceeded for IP address {}. Counted violations: {}", origin, actualAbuses);
+            authentication.disconnect();
+            log.warn("Abuse limit exceeded for IP address {}/Authentication {}. Counted violations: {}",
+                    origin, authentication, actualAbuses);
             return false;
         }
         return true;

src/main/java/de/skuzzle/ghpromexporter/web/AuthenticationProvider.java

@@ -2,6 +2,7 @@
 
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
+import java.util.List;
 
 import org.springframework.http.server.reactive.ServerHttpRequest;
 
@@ -29,6 +30,10 @@ public static GitHubAuthentication parseRequest(ServerHttpRequest request) {
                 return GitHubAuthentication.token(authorization.substring("bearer ".length()));
             }
         }
+        final List<String> tokenParam = request.getQueryParams().get("token");
+        if (tokenParam.size() == 1) {
+            return GitHubAuthentication.token(tokenParam.get(0));
+        }
         return GitHubAuthentication.anonymous(request.getRemoteAddress().getAddress());
     }
 

src/main/java/de/skuzzle/ghpromexporter/web/ScrapeEndpointController.java

@@ -47,7 +47,7 @@ public Mono<ResponseEntity<String>> scrapeRepositories(
         final var targets = MultipleScrapeTargets.parse(owner, repositories);
         log.info("Request from '{}' to scrape '{}'", gitHubAuthentication, targets);
 
-        return abuseLimiter.blockAbusers(origin)
+        return abuseLimiter.blockAbusers(origin, gitHubAuthentication)
                 .flatMap(__ -> scrapeRepositoriesAndSerialize(gitHubAuthentication, targets, contentType))
                 .doOnError(exception -> abuseLimiter.recordFailedCall(exception, origin))
                 .onErrorResume(exception -> Mono.just(ResponseEntity.badRequest().body(exception.getMessage())))

src/test/java/de/skuzzle/ghpromexporter/github/FailingGitHubAuthentication.java

@@ -23,4 +23,9 @@ public GitHub connectToGithub() throws IOException {
     public boolean isAnonymous() {
         return anonymous;
     }
+
+    @Override
+    public void disconnect() {
+        // nothing to do
+    }
 }

src/test/java/de/skuzzle/ghpromexporter/github/MockRepositoryGitHubAuthentication.java

@@ -43,4 +43,9 @@ public boolean isAnonymous() {
         return anonymous;
     }
 
+    @Override
+    public void disconnect() {
+        // nothing to do
+    }
+
 }

swarm-stack/production.yml

@@ -10,6 +10,10 @@ services:
     networks:
     - traefik_proxy
     - monitoring
+    environment:
+      # GH Issue #8: https://github.com/skuzzle/gh-prom-exporter/issues/8
+      # mitigate false positives during abuse detection
+      WEB_ABUSE_LIMIT: '20'
     deploy:
       labels:
         traefik.enable: 'true'