Merge branch 'main' into Update_docs_on_spawning_network_in_k8s

javascript/packages/orchestrator/src/configGenerator.ts

@@ -242,7 +242,9 @@ export async function generateNetworkSpec(
 
   if (config.parachains && config.parachains.length) {
     for (const parachain of config.parachains) {
-      const para: CHAIN = whichChain(parachain.chain || "");
+      const para: CHAIN = whichChain(
+        parachain.chain || parachain.chain_spec_path || "",
+      );
 
       let computedStatePath,
         computedStateCommand,

javascript/packages/orchestrator/src/spawner.ts

@@ -86,7 +86,9 @@ export const spawnNode = async (
     const isAssetHubPolkadot =
       parachain &&
       (parachain.chain?.includes("statemint") ||
-        parachain.chain?.includes("asset-hub-polkadot"));
+        parachain.chain?.includes("asset-hub-polkadot") ||
+        parachain.chainSpecPath?.includes("statemint") ||
+        parachain.chainSpecPath?.includes("asset-hub-polkadot"));
     const keystoreFiles = await generateKeystoreFiles(
       node,
       nodeFilesPath,

scripts/jobs_status_exporter/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3.10-slim
+
+WORKDIR /usr/src/app
+
+ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED 1
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY metrics.py .
+
+CMD ["python", "./metrics.py"]

scripts/jobs_status_exporter/deployment.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: metrics-deployment
+  labels:
+    app: metrics
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: metrics
+  template:
+    metadata:
+      labels:
+        app: metrics
+    spec:
+      containers:
+      - name: metrics
+        image: emamihe/gitlab-runner-metrics:1.0
+        ports:
+        - containerPort: 8000
+          name: http-metrics
+        env:
+        - name: GITLAB_PRIVATE_TOKEN
+          valueFrom:
+            secretKeyRef:
+              name: gitlab-token
+              key: token
+        - name: RUNNER_ID
+          value: "RkrwHxX5"

scripts/jobs_status_exporter/main.py

@@ -0,0 +1,58 @@
+import requests
+import os
+from flask import Flask, Response
+from prometheus_client import Gauge, generate_latest, REGISTRY
+
+GITLAB_URL = "https://gitlab.parity.io"
+PROJECT_IDS = os.environ.get("GITLAB_PROJECT_IDS", "").split(",")
+PRIVATE_TOKEN = os.environ.get("PRIVATE_TOKEN", "")
+TARGET_JOB_NAME = os.environ.get("JOB_NAME", "")
+
+HEADERS = {
+    "Private-Token": PRIVATE_TOKEN
+}
+
+app = Flask(__name__)
+
+job_status_gauge = Gauge('gitlab_pipeline_job_status',
+                         'Number of targeted jobs per status',
+                         ['status', 'project_id'])
+
+def get_all_pages(url):
+    response = requests.get(url, headers=HEADERS)
+    response.raise_for_status()
+
+    data = response.json()
+    yield from data
+
+    while 'next' in response.links.keys():
+        response = requests.get(response.links['next']['url'], headers=HEADERS)
+        response.raise_for_status()
+        data = response.json()
+        yield from data
+
+@app.route('/metrics', methods=['GET'])
+def metrics():
+    for project_id in PROJECT_IDS:
+        statistics = {
+            'canceled': 0,
+            'success': 0,
+            'pending': 0,
+            'running': 0,
+            'failed': 0
+        }
+
+        url = f"{GITLAB_URL}/api/v4/projects/{project_id}/pipelines"
+        for pipeline in get_all_pages(url):
+            jobs_url = f"{GITLAB_URL}/api/v4/projects/{project_id}/pipelines/{pipeline['id']}/jobs"
+            for job in get_all_pages(jobs_url):
+                if job['name'] == TARGET_JOB_NAME and job['status'] in statistics:
+                    statistics[job['status']] += 1
+
+        for status, count in statistics.items():
+            job_status_gauge.labels(status=status, project_id=project_id).set(count)
+
+    return Response(generate_latest(REGISTRY), mimetype="text/plain")
+
+if __name__ == "__main__":
+    app.run(host='0.0.0.0', port=8000)

scripts/jobs_status_exporter/metrics.py

@@ -0,0 +1,48 @@
+import os
+import requests
+import time
+from prometheus_client import start_http_server, Gauge
+
+GITLAB_API_ENDPOINT = 'https://gitlab.parity.io/api/v4/runners/{}/jobs'
+GITLAB_PRIVATE_TOKEN = os.getenv('GITLAB_PRIVATE_TOKEN')
+RUNNER_ID = os.getenv('RUNNER_ID')
+
+if not GITLAB_PRIVATE_TOKEN or not RUNNER_ID:
+    raise EnvironmentError('The environment variables GITLAB_PRIVATE_TOKEN and RUNNER_ID must be set.')
+
+status_gauges = {
+    'created': Gauge('gitlab_runner_jobs_created', 'Number of created jobs'),
+    'pending': Gauge('gitlab_runner_jobs_pending', 'Number of pending jobs'),
+    'running': Gauge('gitlab_runner_jobs_running', 'Number of running jobs'),
+    'failed': Gauge('gitlab_runner_jobs_failed', 'Number of failed jobs'),
+    'success': Gauge('gitlab_runner_jobs_success', 'Number of successful jobs'),
+    'canceled': Gauge('gitlab_runner_jobs_canceled', 'Number of canceled jobs'),
+    'skipped': Gauge('gitlab_runner_jobs_skipped', 'Number of skipped jobs'),
+    'manual': Gauge('gitlab_runner_jobs_manual', 'Number of manual jobs'),
+}
+
+def fetch_jobs_by_runner(runner_id):
+    """Fetch jobs from a specific GitLab Runner and update Prometheus metrics."""
+    headers = {'PRIVATE-TOKEN': GITLAB_PRIVATE_TOKEN}
+    response = requests.get(GITLAB_API_ENDPOINT.format(runner_id), headers=headers)
+    response.raise_for_status()  
+    jobs = response.json()
+
+    for gauge in status_gauges.values():
+        gauge.set(0)
+
+    for job in jobs:
+        status = job.get('status')
+        if status in status_gauges:
+            status_gauges[status].inc()
+
+def main():
+    start_http_server(8000)
+    print("Metrics server running on port 8000")
+
+    while True:
+        fetch_jobs_by_runner(RUNNER_ID)
+        time.sleep(60) 
+
+if __name__ == '__main__':
+    main()

scripts/jobs_status_exporter/requirements.txt

@@ -0,0 +1,3 @@
+Flask==2.0.1
+requests==2.26.0
+prometheus_client==0.11.0

scripts/jobs_status_exporter/runner_id

@@ -0,0 +1 @@
+175886

scripts/jobs_status_exporter/service-monitor.yaml

@@ -0,0 +1,13 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: metrics-servicemonitor
+  labels:
+    app: metrics
+spec:
+  selector:
+    matchLabels:
+      app: metrics
+  endpoints:
+  - port: http-metrics
+    interval: 15s

scripts/jobs_status_exporter/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: metrics-service
+  labels:
+    app: metrics
+spec:
+  selector:
+    app: metrics
+  ports:
+  - port: 8000
+    targetPort: http-metrics