Merge branch 'release/0.0.16'

README.md

@@ -9,15 +9,15 @@
 Export your favorite GitHub repositories to Prometheus
 
 * Use it _as a service_: See https://gh.skuzzle.de for instructions
-* Deploy it _on-premise_: `docker pull ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.15`
+* Deploy it _on-premise_: `docker pull ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.16`
 
 ## On-Premise deployment with docker
 This application can easily be run as a docker container in whatever environment you like:
 
 ```
 docker run -p 8080:8080 \
     -e WEB_ALLOWANONYMOUSSCRAPE=true \
-    ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.15
+    ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.16
 ```
 
 With _anonymous scraping_ allowed, you can now easily view the scrape results directly in the browser by navigating to
@@ -38,7 +38,7 @@ scrape_configs:
 In case you want to enforce authenticated scrapes only, use this configuration instead:
 ```
 docker run -p 8080:8080 \
-    ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.15
+    ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.16
 ```
 
 Scraping now requires a GitHub access token, otherwise the service will respond with 401/Unauthorized. 

RELEASE_NOTES.md

@@ -2,8 +2,9 @@
 
 [![Coverage Status](https://coveralls.io/repos/github/skuzzle/gh-prom-exporter/badge.svg?branch=master)](https://coveralls.io/github/skuzzle/gh-prom-exporter?branch=master) [![Twitter Follow](https://img.shields.io/twitter/follow/skuzzleOSS.svg?style=social)](https://twitter.com/skuzzleOSS)
 
-* Add option to pass GitHub token via query parameter: `?token=...` (Note: this is primarily intended for local testing and should not be used in production)
+* Add option to pass GitHub token via query parameter: `?token=...` (Note: this is primarily intended for local testing and should not be used in production).
+* [#8](https://github.com/skuzzle/gh-prom-exporter/issues/8) Possible fix for scraping while statistics are absent.
 
 ```
-docker pull ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.15
+docker pull ghcr.io/skuzzle/gh-prom-exporter/gh-prom-exporter:0.0.16
 ```

pom.xml

@@ -11,7 +11,7 @@
 
     <groupId>de.skuzzle.ghpromexporter</groupId>
     <artifactId>gh-prom-exporter</artifactId>
-    <version>0.0.15</version>
+    <version>0.0.16</version>
 
     <name>gh-prom-exporter</name>
     <description>Export GitHub repository metrics in prometheus format</description>

readme/RELEASE_NOTES.md

@@ -2,7 +2,8 @@
 
 [![Coverage Status](https://coveralls.io/repos/github/${github.user}/${github.name}/badge.svg?branch=${github.main-branch})](https://coveralls.io/github/${github.user}/${github.name}?branch=${github.main-branch}) [![Twitter Follow](https://img.shields.io/twitter/follow/skuzzleOSS.svg?style=social)](https://twitter.com/skuzzleOSS)
 
-* Add option to pass GitHub token via query parameter: `?token=...` (Note: this is primarily intended for local testing and should not be used in production)
+* Add option to pass GitHub token via query parameter: `?token=...` (Note: this is primarily intended for local testing and should not be used in production).
+* [#8](https://github.com/skuzzle/gh-prom-exporter/issues/8) Possible fix for scraping while statistics are absent.
 
 ```
 docker pull ${docker.image.name}:${project.version}

src/main/java/de/skuzzle/ghpromexporter/github/ScrapableRepository.java

@@ -13,6 +13,8 @@
 import org.kohsuke.github.GHRepositoryStatistics.CodeFrequency;
 import org.kohsuke.github.GHRepositoryStatistics.ContributorStats;
 import org.kohsuke.github.GitHub;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Downloads all relevant information from a github repository.
@@ -21,14 +23,19 @@
  */
 public final class ScrapableRepository {
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(ScrapableRepository.class);
+
     private final GHRepository repository;
     private final List<CodeFrequency> codeFrequency;
     private final int commitsToMainBranch;
+    private final boolean statisticsAvailable;
 
-    private ScrapableRepository(GHRepository repository, List<CodeFrequency> codeFrequency, int commitsToMainBranch) {
+    private ScrapableRepository(GHRepository repository, List<CodeFrequency> codeFrequency, int commitsToMainBranch,
+            boolean statisticsAvailable) {
         this.repository = repository;
         this.codeFrequency = codeFrequency;
         this.commitsToMainBranch = commitsToMainBranch;
+        this.statisticsAvailable = statisticsAvailable;
     }
 
     public static ScrapableRepository load(GitHubAuthentication authentication, String repositoryFullName) {
@@ -42,20 +49,73 @@ public static ScrapableRepository load(GitHubAuthentication authentication, Stri
             // getStatistics call
             // Until the statistics are available, getCodeFrequency and
             // getContributorStats throw NPE, which might lead to false positives during
-            // abuse detection. To mitigate this, abuse detection must be either disabled
-            // or threshold must be set to a high value.
-            final GHRepositoryStatistics statistics = repository.getStatistics();
-            final List<CodeFrequency> codeFrequency = statistics.getCodeFrequency();
-            final IntSummaryStatistics commitsToMainBranch = StreamSupport
-                    .stream(statistics.getContributorStats().spliterator(), false)
-                    .collect(summarizingInt(ContributorStats::getTotal));
-
-            return new ScrapableRepository(repository, codeFrequency, (int) commitsToMainBranch.getSum());
+            // abuse detection.
+            final Statistics statistics = new Statistics(repository.getStatistics());
+            final List<CodeFrequency> codeFrequency = statistics.codeFrequency();
+            final int commitsToMainBranch = statistics.commitsToMainBranch();
+
+            return new ScrapableRepository(repository, codeFrequency, commitsToMainBranch, statistics.available());
         } catch (final IOException e) {
             throw new UncheckedIOException(e);
-        } catch (final InterruptedException e) {
-            Thread.currentThread().interrupt();
-            throw new IllegalStateException(e);
+        }
+    }
+
+    /**
+     * GH Issue #8: https://github.com/skuzzle/gh-prom-exporter/issues/8
+     * <p>
+     * Internal anti corruption layer class to obtain repository statistics. GitHub API
+     * implementation occasionally throws NPE while statistics are being generated
+     * asynchronously by GitHub. This class gracefully replaces those NPEs with default
+     * values.
+     */
+    private static final class Statistics {
+
+        private final GHRepositoryStatistics statistics;
+        private boolean available = true;
+
+        private Statistics(GHRepositoryStatistics statistics) {
+            this.statistics = statistics;
+        }
+
+        boolean available() {
+            return available;
+        }
+
+        List<CodeFrequency> codeFrequency() {
+            if (!available) {
+                return List.of();
+            }
+
+            try {
+                return statistics.getCodeFrequency();
+            } catch (final Exception e) {
+                available = false;
+                LOGGER.error("Could not obtain repository code frequency", e);
+            }
+            return List.of();
+        }
+
+        int commitsToMainBranch() {
+            if (!available) {
+                return 0;
+            }
+            Exception exception = null;
+            try {
+                final IntSummaryStatistics commitsToMainBranch = StreamSupport
+                        .stream(statistics.getContributorStats().spliterator(), false)
+                        .collect(summarizingInt(ContributorStats::getTotal));
+
+                return (int) commitsToMainBranch.getSum();
+            } catch (final InterruptedException e) {
+                exception = e;
+                Thread.currentThread().interrupt();
+            } catch (final Exception e) {
+                exception = e;
+            }
+
+            LOGGER.error("Could not obtain repository contributor stats", exception);
+            available = false;
+            return 0;
         }
     }
 
@@ -83,6 +143,10 @@ public int sizeInKb() {
         return repository.getSize();
     }
 
+    public boolean statisticsAvailable() {
+        return statisticsAvailable;
+    }
+
     public long totalAdditions() {
         return codeFrequency.stream().mapToLong(CodeFrequency::getAdditions).sum();
     }

src/main/java/de/skuzzle/ghpromexporter/scrape/PrometheusRepositoryMetricAggration.java

@@ -59,9 +59,13 @@ private PrometheusRepositoryMetricAggration() {
     public PrometheusRepositoryMetricAggration addRepositoryScrapeResults(
             ScrapeTarget repository,
             ScrapeResult metrics) {
-        additions.labels(repository.owner(), repository.name()).inc(metrics.totalAdditions());
-        deletions.labels(repository.owner(), repository.name()).inc(metrics.totalDeletions());
-        commitsToMainBranch.labels(repository.owner(), repository.name()).inc(metrics.commitsToMainBranch());
+
+        if (metrics.statisticsAvailable()) {
+            additions.labels(repository.owner(), repository.name()).inc(metrics.totalAdditions());
+            deletions.labels(repository.owner(), repository.name()).inc(metrics.totalDeletions());
+            commitsToMainBranch.labels(repository.owner(), repository.name()).inc(metrics.commitsToMainBranch());
+        }
+
         stargazers.labels(repository.owner(), repository.name()).inc(metrics.stargazersCount());
         forks.labels(repository.owner(), repository.name()).inc(metrics.forkCount());
         open_issues.labels(repository.owner(), repository.name()).inc(metrics.openIssueCount());

src/main/java/de/skuzzle/ghpromexporter/scrape/ScrapeResult.java

@@ -15,6 +15,7 @@ public record ScrapeResult(
         int subscriberCount,
         int watchersCount,
         int sizeInKb,
+        boolean statisticsAvailable,
 
         /* Note: this field must always occur last */
         long scrapeDuration) {}

src/main/java/de/skuzzle/ghpromexporter/scrape/ScrapeService.java

@@ -43,6 +43,8 @@ public ScrapeResult scrape(GitHubAuthentication authentication, ScrapeTarget tar
                     scrapableRepository.subscriberCount(),
                     scrapableRepository.watchersCount(),
                     scrapableRepository.sizeInKb(),
+                    scrapableRepository.statisticsAvailable(),
+
                     System.currentTimeMillis() - start);
 
             log.debug("Scraped fresh metrics for {} in {}ms", target, repositoryMetrics.scrapeDuration());

src/test/java/de/skuzzle/ghpromexporter/github/MockRepositoryBuilder.java

@@ -95,6 +95,20 @@ public MockRepositoryBuilder withDeletions(long deletions) {
         return this;
     }
 
+    // simulates behavior described in GH Issue #8:
+    // https://github.com/skuzzle/gh-prom-exporter/issues/8
+    public MockRepositoryBuilder withThrowingCodeFrequency() throws Exception {
+        when(statistics.getCodeFrequency()).thenThrow(NullPointerException.class);
+        return this;
+    }
+
+    // simulates behavior described in GH Issue #8:
+    // https://github.com/skuzzle/gh-prom-exporter/issues/8
+    public MockRepositoryBuilder withThrowingContributorStats() throws Exception {
+        when(statistics.getContributorStats()).thenThrow(NullPointerException.class);
+        return this;
+    }
+
     public GHRepository build() {
         return this.repository;
     }

src/test/java/de/skuzzle/ghpromexporter/web/ScrapeEndpointControllerTest.java

@@ -100,6 +100,44 @@ void test_successful_initial_scrape(Snapshot snapshot) throws Exception {
         });
     }
 
+    @Test
+    void test_scrape_with_missing_contributor_stats(Snapshot snapshot) throws Exception {
+        final var serviceCall = testClient.getStatsFor("skuzzle", "test-repo");
+        final GitHubAuthentication gitHubAuthentication = successfulAuthenticationForRepository(
+                withName("skuzzle", "test-repo")
+                        .withThrowingContributorStats());
+
+        authentication.with(gitHubAuthentication, () -> {
+
+            StepVerifier.create(serviceCall)
+                    .assertNext(response -> {
+                        snapshot.assertThat(response.getBody())
+                                .as(canonicalPrometheusRegistry())
+                                .matchesSnapshotText();
+                    })
+                    .verifyComplete();
+        });
+    }
+
+    @Test
+    void test_scrape_with_missing_code_frequency(Snapshot snapshot) throws Exception {
+        final var serviceCall = testClient.getStatsFor("skuzzle", "test-repo");
+        final GitHubAuthentication gitHubAuthentication = successfulAuthenticationForRepository(
+                withName("skuzzle", "test-repo")
+                        .withThrowingCodeFrequency());
+
+        authentication.with(gitHubAuthentication, () -> {
+
+            StepVerifier.create(serviceCall)
+                    .assertNext(response -> {
+                        snapshot.assertThat(response.getBody())
+                                .as(canonicalPrometheusRegistry())
+                                .matchesSnapshotText();
+                    })
+                    .verifyComplete();
+        });
+    }
+
     @Test
     void test_successful_scrape_open_metrics(Snapshot snapshot) throws Exception {
         final var serviceCall = testClient.getStatsFor("skuzzle", "test-repo",

src/test/java/de/skuzzle/ghpromexporter/web/TestClient.java

@@ -2,7 +2,7 @@
 
 import java.util.Map;
 
-import org.springframework.boot.web.server.LocalServerPort;
+import org.springframework.boot.test.web.server.LocalServerPort;
 import org.springframework.context.annotation.Lazy;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.ResponseEntity;

src/test/resources/de/skuzzle/ghpromexporter/web/ScrapeEndpointControllerTest_snapshots/test_scrape_with_missing_code_frequency_0.snapshot

@@ -0,0 +1,49 @@
+dynamic-directory: false
+snapshot-name: test_scrape_with_missing_code_frequency_0
+snapshot-number: 0
+test-class: de.skuzzle.ghpromexporter.web.ScrapeEndpointControllerTest
+test-method: test_scrape_with_missing_code_frequency
+
+INFO: This registry is sorted and cleaned up from some random values to ensure deterministic testing behavior
+
+# HELP github_additions_total Sum of additions over the last 52 weeks
+# HELP github_commits_to_main_branch_total Number of commits to the main branch
+# HELP github_deletions_total Negative sum of deletions over the last 52 weeks
+# HELP github_forks_created The repository's fork count
+# HELP github_forks_total The repository's fork count
+# HELP github_open_issues_created The repository's open issue count
+# HELP github_open_issues_total The repository's open issue count
+# HELP github_scrape_duration Duration of a single scrape
+# HELP github_scrape_duration_created Duration of a single scrape
+# HELP github_size_created The repository's size in KB
+# HELP github_size_total The repository's size in KB
+# HELP github_stargazers_created The repository's stargazer count
+# HELP github_stargazers_total The repository's stargazer count
+# HELP github_subscribers_created The repository's subscriber count
+# HELP github_subscribers_total The repository's subscriber count
+# HELP github_watchers_created The repository's watcher count
+# HELP github_watchers_total The repository's watcher count
+# TYPE github_additions_total counter
+# TYPE github_commits_to_main_branch_total counter
+# TYPE github_deletions_total counter
+# TYPE github_forks_created gauge
+# TYPE github_forks_total counter
+# TYPE github_open_issues_created gauge
+# TYPE github_open_issues_total counter
+# TYPE github_scrape_duration summary
+# TYPE github_scrape_duration_created gauge
+# TYPE github_size_created gauge
+# TYPE github_size_total counter
+# TYPE github_stargazers_created gauge
+# TYPE github_stargazers_total counter
+# TYPE github_subscribers_created gauge
+# TYPE github_subscribers_total counter
+# TYPE github_watchers_created gauge
+# TYPE github_watchers_total counter
+github_forks_total{owner="skuzzle",repository="test-repo",} 0.0
+github_open_issues_total{owner="skuzzle",repository="test-repo",} 0.0
+github_scrape_duration_count{owner="skuzzle",repository="test-repo",} 1.0
+github_size_total{owner="skuzzle",repository="test-repo",} 0.0
+github_stargazers_total{owner="skuzzle",repository="test-repo",} 0.0
+github_subscribers_total{owner="skuzzle",repository="test-repo",} 0.0
+github_watchers_total{owner="skuzzle",repository="test-repo",} 0.0

src/test/resources/de/skuzzle/ghpromexporter/web/ScrapeEndpointControllerTest_snapshots/test_scrape_with_missing_contributor_stats_0.snapshot

@@ -0,0 +1,49 @@
+dynamic-directory: false
+snapshot-name: test_scrape_with_missing_contributor_stats_0
+snapshot-number: 0
+test-class: de.skuzzle.ghpromexporter.web.ScrapeEndpointControllerTest
+test-method: test_scrape_with_missing_contributor_stats
+
+INFO: This registry is sorted and cleaned up from some random values to ensure deterministic testing behavior
+
+# HELP github_additions_total Sum of additions over the last 52 weeks
+# HELP github_commits_to_main_branch_total Number of commits to the main branch
+# HELP github_deletions_total Negative sum of deletions over the last 52 weeks
+# HELP github_forks_created The repository's fork count
+# HELP github_forks_total The repository's fork count
+# HELP github_open_issues_created The repository's open issue count
+# HELP github_open_issues_total The repository's open issue count
+# HELP github_scrape_duration Duration of a single scrape
+# HELP github_scrape_duration_created Duration of a single scrape
+# HELP github_size_created The repository's size in KB
+# HELP github_size_total The repository's size in KB
+# HELP github_stargazers_created The repository's stargazer count
+# HELP github_stargazers_total The repository's stargazer count
+# HELP github_subscribers_created The repository's subscriber count
+# HELP github_subscribers_total The repository's subscriber count
+# HELP github_watchers_created The repository's watcher count
+# HELP github_watchers_total The repository's watcher count
+# TYPE github_additions_total counter
+# TYPE github_commits_to_main_branch_total counter
+# TYPE github_deletions_total counter
+# TYPE github_forks_created gauge
+# TYPE github_forks_total counter
+# TYPE github_open_issues_created gauge
+# TYPE github_open_issues_total counter
+# TYPE github_scrape_duration summary
+# TYPE github_scrape_duration_created gauge
+# TYPE github_size_created gauge
+# TYPE github_size_total counter
+# TYPE github_stargazers_created gauge
+# TYPE github_stargazers_total counter
+# TYPE github_subscribers_created gauge
+# TYPE github_subscribers_total counter
+# TYPE github_watchers_created gauge
+# TYPE github_watchers_total counter
+github_forks_total{owner="skuzzle",repository="test-repo",} 0.0
+github_open_issues_total{owner="skuzzle",repository="test-repo",} 0.0
+github_scrape_duration_count{owner="skuzzle",repository="test-repo",} 1.0
+github_size_total{owner="skuzzle",repository="test-repo",} 0.0
+github_stargazers_total{owner="skuzzle",repository="test-repo",} 0.0
+github_subscribers_total{owner="skuzzle",repository="test-repo",} 0.0
+github_watchers_total{owner="skuzzle",repository="test-repo",} 0.0