From 89bb54c797f21a9d5d8734632db4bb0a812da324 Mon Sep 17 00:00:00 2001 From: Ethan Wu Date: Wed, 19 Oct 2022 21:14:35 -0700 Subject: [PATCH 01/12] chore: update desktop dpi (#1326) Co-authored-by: Ethan Wu --- hieradata/nodes/arsenic.yaml | 1 + hieradata/nodes/bigbang.yaml | 1 + hieradata/nodes/blizzard.yaml | 1 + hieradata/nodes/chaos.yaml | 1 + hieradata/nodes/famine.yaml | 1 + hieradata/nodes/meteorstorm.yaml | 1 + hieradata/nodes/surge.yaml | 1 + hieradata/nodes/wildfire.yaml | 1 + 8 files changed, 8 insertions(+) create mode 100644 hieradata/nodes/arsenic.yaml create mode 100644 hieradata/nodes/bigbang.yaml create mode 100644 hieradata/nodes/chaos.yaml create mode 100644 hieradata/nodes/famine.yaml create mode 100644 hieradata/nodes/meteorstorm.yaml create mode 100644 hieradata/nodes/surge.yaml create mode 100644 hieradata/nodes/wildfire.yaml diff --git a/hieradata/nodes/arsenic.yaml b/hieradata/nodes/arsenic.yaml new file mode 100644 index 000000000..d63e45eca --- /dev/null +++ b/hieradata/nodes/arsenic.yaml @@ -0,0 +1 @@ +ocf_desktop::xsession::scale: 1.5 diff --git a/hieradata/nodes/bigbang.yaml b/hieradata/nodes/bigbang.yaml new file mode 100644 index 000000000..d63e45eca --- /dev/null +++ b/hieradata/nodes/bigbang.yaml @@ -0,0 +1 @@ +ocf_desktop::xsession::scale: 1.5 diff --git a/hieradata/nodes/blizzard.yaml b/hieradata/nodes/blizzard.yaml index 488227e65..1c5f8418c 100644 --- a/hieradata/nodes/blizzard.yaml +++ b/hieradata/nodes/blizzard.yaml @@ -1 +1,2 @@ opstaff: true +ocf_desktop::xsession::scale: 1.5 diff --git a/hieradata/nodes/chaos.yaml b/hieradata/nodes/chaos.yaml new file mode 100644 index 000000000..d63e45eca --- /dev/null +++ b/hieradata/nodes/chaos.yaml @@ -0,0 +1 @@ +ocf_desktop::xsession::scale: 1.5 diff --git a/hieradata/nodes/famine.yaml b/hieradata/nodes/famine.yaml new file mode 100644 index 000000000..d63e45eca --- /dev/null +++ b/hieradata/nodes/famine.yaml @@ -0,0 +1 @@ +ocf_desktop::xsession::scale: 1.5 diff --git a/hieradata/nodes/meteorstorm.yaml b/hieradata/nodes/meteorstorm.yaml new file mode 100644 index 000000000..d63e45eca --- /dev/null +++ b/hieradata/nodes/meteorstorm.yaml @@ -0,0 +1 @@ +ocf_desktop::xsession::scale: 1.5 diff --git a/hieradata/nodes/surge.yaml b/hieradata/nodes/surge.yaml new file mode 100644 index 000000000..d63e45eca --- /dev/null +++ b/hieradata/nodes/surge.yaml @@ -0,0 +1 @@ +ocf_desktop::xsession::scale: 1.5 diff --git a/hieradata/nodes/wildfire.yaml b/hieradata/nodes/wildfire.yaml new file mode 100644 index 000000000..d63e45eca --- /dev/null +++ b/hieradata/nodes/wildfire.yaml @@ -0,0 +1 @@ +ocf_desktop::xsession::scale: 1.5 From 99f7b85e321d6821b9831c5d003394889bda7803 Mon Sep 17 00:00:00 2001 From: Ethan Wu Date: Thu, 20 Oct 2022 11:38:11 -0700 Subject: [PATCH 02/12] fix: add ewdc ip ranges to dns acl (#1328) --- modules/ocf_ns/templates/named.conf.options.erb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/ocf_ns/templates/named.conf.options.erb b/modules/ocf_ns/templates/named.conf.options.erb index 875f0fe05..5782d79fe 100644 --- a/modules/ocf_ns/templates/named.conf.options.erb +++ b/modules/ocf_ns/templates/named.conf.options.erb @@ -2,6 +2,8 @@ acl "ocf" { 169.229.226.0/24; 2607:f140:8801::/48; + 169.229.220.64/28; + 2607:f140:0:32::/64; }; acl "ucb" { From 1fc0352a586f7b2284496e2fe73928c0a5d4b739 Mon Sep 17 00:00:00 2001 From: Ethan Wu Date: Thu, 20 Oct 2022 11:38:25 -0700 Subject: [PATCH 03/12] feat: remove FF newtab annoyances (#1327) --- modules/ocf/templates/firefox/prefs.js.erb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/modules/ocf/templates/firefox/prefs.js.erb b/modules/ocf/templates/firefox/prefs.js.erb index 6d04a6c74..310f89aab 100644 --- a/modules/ocf/templates/firefox/prefs.js.erb +++ b/modules/ocf/templates/firefox/prefs.js.erb @@ -3,8 +3,12 @@ pref("intl.locale.requested", ""); pref("browser.startup.homepage", "<%= @browser_homepage %>", locked); pref("browser.cache.disk.capacity", 0); pref("browser.download.useDownloadDir", false); +pref("browser.newtabpage.activity-stream.discoverystream.enabled", false); pref("browser.newtabpage.activity-stream.feeds.section.topstories", false); +pref("browser.newtabpage.activity-stream.feeds.topsites", false); pref("browser.newtabpage.activity-stream.showSponsored", false); +pref("browser.newtabpage.activity-stream.showSponsoredTopSite", false); +pref("browser.newtabpage.activity-stream.telemetry", false); pref("browser.privatebrowsing.autostart", true); pref("browser.search.geoSpecificDefaults", false); pref("browser.shell.checkDefaultBrowser", false); From 8c6c1dce9f95ece41a23b76ba087115c18b3fe8b Mon Sep 17 00:00:00 2001 From: Kalissaac Date: Fri, 21 Oct 2022 16:05:47 -0700 Subject: [PATCH 04/12] Remove staff meeting shorturls (#1331) All staff meeting shorturls have been migrated from ocf.io/* to ocf.io/s/* --- modules/ocf_www/manifests/site/shorturl.pp | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/modules/ocf_www/manifests/site/shorturl.pp b/modules/ocf_www/manifests/site/shorturl.pp index b78ce6e4b..d76fa8961 100644 --- a/modules/ocf_www/manifests/site/shorturl.pp +++ b/modules/ocf_www/manifests/site/shorturl.pp @@ -49,10 +49,6 @@ {rewrite_rule => '^/donate$ https://give.berkeley.edu/egiving/index.cfm?fund=FU1450000&org=Open+Computing+Facility&amt=&fundlist=FU1450000 [R]'}, {rewrite_rule => '^/email-update$ https://status.ocf.berkeley.edu/2014/06/email-discontinuation-update-forward.html [R]'}, {rewrite_rule => '^/eligibility$ https://www.ocf.berkeley.edu/docs/membership/eligibility/ [R]'}, - {rewrite_rule => '^/fa22-?mt2$ https://docs.google.com/presentation/d/1X4zZYtCJ5t86s1FJu1XGg2RI08-kauJw11y02_UESNg/edit?usp=sharing [R]'}, - {rewrite_rule => '^/fa22-?mt3$ https://docs.google.com/presentation/d/1qyVDOZtGs7HzvRAIh4Nk7ZrOuU_GDAHdo_yBnLn1_1E/edit?usp=sharing [R]'}, - {rewrite_rule => '^/fa22-?mt4$ https://docs.google.com/presentation/d/1F1ZM8QWnxeNL-7rysLUmtp2z3SCyQMGZzpa3CGj7VEM/edit?usp=sharing [R]'}, - {rewrite_rule => '^/fa22-?mt6$ https://docs.google.com/presentation/d/1jxGrECJKDiRzacFXPHKrNcHyoVSNv7OhYka1M1ioHwU/edit?usp=sharing [R]'}, {rewrite_rule => '^/families$ https://docs.google.com/presentation/d/1y49eQj-SPIsMtIio2KFx86SF6tKcv1yB16pO6AS2uk4/edit [R]'}, {rewrite_rule => '^/faq$ https://www.ocf.berkeley.edu/docs/faq/ [R]'}, {rewrite_rule => '^/gh/([^/]*)(/(?!blob/)(?!tree/)(?!info/)(?!issue)(?!pull)(?!search).+)$ https://ocf.io/gh/$1/blob/master$2 [R]'}, @@ -140,14 +136,6 @@ {rewrite_rule => '^/social$ https://docs.google.com/forms/d/e/1FAIpQLSdLUhDYQug53NXdPEeJkLzB9JNUMUWJmXMoRSv18PkhEh4h4Q/viewform [R]'}, {rewrite_rule => '^/social-?rules$ https://www.recurse.com/social-rules [R]'}, {rewrite_rule => '^/socialbingo$ https://docs.google.com/forms/d/e/1FAIpQLSeU2iZf_SnephTu6KQ_VfeFBI-YMetTSioeibFKwgtoITlG-w/viewform [R]'}, - {rewrite_rule => '^/sp22-?mt2$ https://ocf.io/sp22-mt3 [R]'}, - {rewrite_rule => '^/sp22-?mt3$ https://docs.google.com/presentation/d/12itQ4l3We-cGltobBtqYXFlC4m-SbNrZrBQ_FlylWHQ/edit?usp=sharing [R]'}, - {rewrite_rule => '^/sp22-?mt5$ https://docs.google.com/presentation/d/1oYyXc4W8Q9Xii4ToO1V0gVU27z7cdq8IvLs_vMMUm1Y/edit?usp=sharing [R]'}, - {rewrite_rule => '^/sp22-?mt6$ https://docs.google.com/presentation/d/1f3k4_OA3t92Fug-mwxP6fwO8Xt_r9mqwibdTW6rsWeI/edit?usp=sharing [R]'}, - {rewrite_rule => '^/sp22-?mt7$ https://docs.google.com/presentation/d/1qXn8_mL0F6w59kdHYK4TThZ2_UovysW24OmdD2yjTNo/edit?usp=sharing [R]'}, - {rewrite_rule => '^/sp22-?mt8$ https://docs.google.com/presentation/d/18pqxU5O0HYVMqdYLmUU4yaCjZw1iH8Zcr7ICse7L1XI/edit?usp=sharing [R]'}, - {rewrite_rule => '^/sp22-?mt9$ https://docs.google.com/presentation/d/1i9fcSF1_7Z4bHy-PqpuK-VFDVVJZ8-Ha29pjI8LEhj4/edit?usp=sharing [R]'}, - {rewrite_rule => '^/sp22-?mt11$ https://docs.google.com/presentation/d/1ZtqObh5t_-KkYJLf3TqaUmxOpNONqv3SypW-pYcXOZA/edit?usp=sharing [R]'}, {rewrite_rule => '^/ssh$ https://www.ocf.berkeley.edu/docs/services/shell/ [R]'}, {rewrite_rule => '^/staff$ https://www.ocf.berkeley.edu/about/staff [R]'}, {rewrite_rule => '^/staffhours$ https://ocf.io/staff-hours [R]'}, From f977fa60479843483703cf4690e198b5dfe46644 Mon Sep 17 00:00:00 2001 From: Bryan Li Date: Sun, 23 Oct 2022 16:30:20 -0700 Subject: [PATCH 05/12] Elevated priority for ingress nginx pods. (#1332) --- .../ocf_kubernetes/files/ingress_deploy.yaml | 659 ++++++++++++++++++ .../manifests/master/ingress/nginx.pp | 8 +- 2 files changed, 664 insertions(+), 3 deletions(-) create mode 100644 modules/ocf_kubernetes/files/ingress_deploy.yaml diff --git a/modules/ocf_kubernetes/files/ingress_deploy.yaml b/modules/ocf_kubernetes/files/ingress_deploy.yaml new file mode 100644 index 000000000..31589db7a --- /dev/null +++ b/modules/ocf_kubernetes/files/ingress_deploy.yaml @@ -0,0 +1,659 @@ + +apiVersion: v1 +kind: Namespace +metadata: + name: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + +--- +# Source: ingress-nginx/templates/controller-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +automountServiceAccountToken: true +--- +# Source: ingress-nginx/templates/controller-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +data: + allow-snippet-annotations: 'true' +--- +# Source: ingress-nginx/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch + - apiGroups: + - '' + resources: + - nodes + verbs: + - get + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch +--- +# Source: ingress-nginx/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +rules: + - apiGroups: + - '' + resources: + - namespaces + verbs: + - get + - apiGroups: + - '' + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - extensions + - networking.k8s.io # k8s 1.14+ + resources: + - ingresses/status + verbs: + - update + - apiGroups: + - networking.k8s.io # k8s 1.14+ + resources: + - ingressclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - configmaps + resourceNames: + - ingress-controller-leader-nginx + verbs: + - get + - update + - apiGroups: + - '' + resources: + - configmaps + verbs: + - create + - apiGroups: + - '' + resources: + - events + verbs: + - create + - patch +--- +# Source: ingress-nginx/templates/controller-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: + - kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/controller-service-webhook.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + type: ClusterIP + ports: + - name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-service.yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + type: NodePort + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + - name: https + port: 443 + protocol: TCP + targetPort: https + selector: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller +--- +# Source: ingress-nginx/templates/controller-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + selector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + revisionHistoryLimit: 10 + minReadySeconds: 0 + template: + metadata: + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/component: controller + spec: + dnsPolicy: ClusterFirst + containers: + - name: controller + image: k8s.gcr.io/ingress-nginx/controller:v0.50.0@sha256:f46fc2d161c97a9d950635acb86fb3f8d4adcfb03ee241ea89c6cde16aa3fdf8 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + args: + - /nginx-ingress-controller + - --election-id=ingress-controller-leader + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + securityContext: + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + runAsUser: 101 + allowPrivilegeEscalation: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + ports: + - name: http + containerPort: 80 + protocol: TCP + - name: https + containerPort: 443 + protocol: TCP + - name: webhook + containerPort: 8443 + protocol: TCP + volumeMounts: + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + resources: + requests: + cpu: 100m + memory: 90Mi + priorityClassName: system-cluster-critical + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: ingress-nginx + terminationGracePeriodSeconds: 300 + volumes: + - name: webhook-cert + secret: + secretName: ingress-nginx-admission +--- +# Source: ingress-nginx/templates/admission-webhooks/validating-webhook.yaml +# before changing this value, check the required kubernetes version +# https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/#prerequisites +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + name: ingress-nginx-admission +webhooks: + - name: validate.nginx.ingress.kubernetes.io + matchPolicy: Equivalent + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + namespace: ingress-nginx + name: ingress-nginx-controller-admission + path: /networking/v1beta1/ingresses +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ingress-nginx-admission + namespace: ingress-nginx + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ingress-nginx-admission + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ingress-nginx-admission + namespace: ingress-nginx + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +rules: + - apiGroups: + - '' + resources: + - secrets + verbs: + - get + - create +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ingress-nginx-admission + namespace: ingress-nginx + annotations: + helm.sh/hook: pre-install,pre-upgrade,post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: + - kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-createSecret.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-create + namespace: ingress-nginx + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +spec: + template: + metadata: + name: ingress-nginx-admission-create + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: create + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + nodeSelector: + kubernetes.io/os: linux + securityContext: + runAsNonRoot: true + runAsUser: 2000 +--- +# Source: ingress-nginx/templates/admission-webhooks/job-patch/job-patchWebhook.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: ingress-nginx-admission-patch + namespace: ingress-nginx + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook +spec: + template: + metadata: + name: ingress-nginx-admission-patch + labels: + helm.sh/chart: ingress-nginx-3.41.0 + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/version: 0.51.0 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admission-webhook + spec: + containers: + - name: patch + image: docker.io/jettech/kube-webhook-certgen:v1.5.1 + imagePullPolicy: IfNotPresent + args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission + nodeSelector: + kubernetes.io/os: linux + securityContext: + runAsNonRoot: true + runAsUser: 2000 diff --git a/modules/ocf_kubernetes/manifests/master/ingress/nginx.pp b/modules/ocf_kubernetes/manifests/master/ingress/nginx.pp index 7b6581401..0491e7b9e 100644 --- a/modules/ocf_kubernetes/manifests/master/ingress/nginx.pp +++ b/modules/ocf_kubernetes/manifests/master/ingress/nginx.pp @@ -2,8 +2,6 @@ $kubernetes_worker_nodes = lookup('kubernetes::worker_nodes') $kubernetes_workers_ipv4 = $kubernetes_worker_nodes.map |$worker| { ldap_attr($worker, 'ipHostNumber') } - $nginx_version = lookup('kubernetes::nginx_version') - $ingress_nginx_url = "https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v${nginx_version}/deploy/static/provider/baremetal/deploy.yaml" file { default: @@ -16,11 +14,15 @@ '/etc/ocf-kubernetes/manifests/ingress/ingress-expose.yaml': content => template('ocf_kubernetes/ingress/ingress_expose.yaml.erb'), mode => '0644'; + + '/etc/ocf-kubernetes/manifests/ingress/ingress-deploy.yaml': + content => 'puppet:///modules/ocf_kubernetes/ingress_deploy.yaml', + mode => '0644'; } # Add ingress-nginx to the cluster ocf_kubernetes::apply { 'ingress-init': - target => $ingress_nginx_url + target => '/etc/ocf-kubernetes/manifests/ingress/ingress-deploy.yaml', } -> # Set up a NodePort service so all kubernetes workers From 1777c78374d8b6cb7b5f4eff09c67f3d10839ac9 Mon Sep 17 00:00:00 2001 From: Ethan Wu Date: Mon, 24 Oct 2022 00:42:35 -0700 Subject: [PATCH 06/12] feat: use sha512 for dns tsig key (#1329) bind9 apparently supports SHA-based algorithms now --- .../ocf/templates/ssl/dehydrated-hook-ddns-tsig.conf.erb | 2 +- modules/ocf_ns/templates/named.conf.options.erb | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/modules/ocf/templates/ssl/dehydrated-hook-ddns-tsig.conf.erb b/modules/ocf/templates/ssl/dehydrated-hook-ddns-tsig.conf.erb index 93b27580e..7294c95a0 100644 --- a/modules/ocf/templates/ssl/dehydrated-hook-ddns-tsig.conf.erb +++ b/modules/ocf/templates/ssl/dehydrated-hook-ddns-tsig.conf.erb @@ -5,5 +5,5 @@ verbosity = 1 # not seem to be actually documented anywhere key_name = letsencrypt.ocf.io key_secret = "<%= @letsencrypt_ddns_key -%>" -key_algorithm = hmac-md5 +key_algorithm = hmac-sha512 dns_rewrite = s/^_acme-challenge\.(.*)(ocf\.berkeley\.edu|ocf\.io)$/\1letsencrypt.ocf.io/ diff --git a/modules/ocf_ns/templates/named.conf.options.erb b/modules/ocf_ns/templates/named.conf.options.erb index 5782d79fe..1ec0ac3fe 100644 --- a/modules/ocf_ns/templates/named.conf.options.erb +++ b/modules/ocf_ns/templates/named.conf.options.erb @@ -61,11 +61,6 @@ zone "multi.uribl.com" { }; key "letsencrypt.ocf.io" { - // Unfortunately bind9 does not appear to support algorithms stronger than - // hmac-md5 for user keys (used for dynamic DNS), but we can make the key - // size fairly large at least. HMAC with md5 does not suffer from the - // collision problems that md5 has, but ideally this would still be changed - // to something stronger in the future. - algorithm HMAC-MD5; + algorithm HMAC-SHA512; secret "<%= @letsencrypt_ddns_key -%>"; }; From 2b003cf842ebf7db0634600bfac32445c257ce01 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 24 Oct 2022 03:51:39 -0400 Subject: [PATCH 07/12] fix: increase kerberos ticket validity for backup script (#1334) --- modules/ocf_backups/files/rsnapshot.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_backups/files/rsnapshot.conf b/modules/ocf_backups/files/rsnapshot.conf index 23739a2c6..d6c86fa17 100644 --- a/modules/ocf_backups/files/rsnapshot.conf +++ b/modules/ocf_backups/files/rsnapshot.conf @@ -21,7 +21,7 @@ cmd_ssh /usr/bin/ssh cmd_logger /usr/bin/logger # remote backups require login as ocfbackups, then `sudo rsync-no-vanished' -cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_preexec /usr/bin/kinit -l 12h -t /opt/share/backups/ocfbackups.keytab ocfbackups cmd_postexec /usr/bin/kdestroy # default is "--delete --numeric-ids --relative --delete-excluded" From c80087268ba05cc0c28f01dfcf79ec2c3b09f706 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 24 Oct 2022 03:07:46 -0700 Subject: [PATCH 08/12] Revert "fix: increase kerberos ticket validity for backup script (#1334)" This reverts commit 2b003cf842ebf7db0634600bfac32445c257ce01. Upon a closer look, the ticket is destroyed upon logout. The default validity for the ocfbackups principal is 1 day. --- modules/ocf_backups/files/rsnapshot.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/ocf_backups/files/rsnapshot.conf b/modules/ocf_backups/files/rsnapshot.conf index d6c86fa17..23739a2c6 100644 --- a/modules/ocf_backups/files/rsnapshot.conf +++ b/modules/ocf_backups/files/rsnapshot.conf @@ -21,7 +21,7 @@ cmd_ssh /usr/bin/ssh cmd_logger /usr/bin/logger # remote backups require login as ocfbackups, then `sudo rsync-no-vanished' -cmd_preexec /usr/bin/kinit -l 12h -t /opt/share/backups/ocfbackups.keytab ocfbackups +cmd_preexec /usr/bin/kinit -t /opt/share/backups/ocfbackups.keytab ocfbackups cmd_postexec /usr/bin/kdestroy # default is "--delete --numeric-ids --relative --delete-excluded" From 012c9ab8edc52af6c3bdcd3c0332d8b8711258e6 Mon Sep 17 00:00:00 2001 From: Nikhil Jha Date: Mon, 24 Oct 2022 17:39:07 -0700 Subject: [PATCH 09/12] remove jaws (also some munin config) (#1323) * delete old config * remove munin --- hieradata/nodes/dataloss.yaml | 1 - hieradata/nodes/hal.yaml | 1 - hieradata/nodes/jaws.yaml | 12 ---- hieradata/nodes/pandemic.yaml | 1 - hieradata/nodes/riptide.yaml | 1 - modules/ocf/files/munin/munin-node.conf | 58 ------------------- modules/ocf/manifests/init.pp | 1 - modules/ocf/manifests/munin/node.pp | 25 -------- modules/ocf/manifests/munin/plugin.pp | 38 ------------ modules/ocf_backups/files/rsnapshot.conf | 3 - modules/ocf_csgo/files/bin/update-csgo | 2 - modules/ocf_csgo/files/etc/csgo-update.cmd | 18 ------ modules/ocf_csgo/files/munin | 37 ------------ modules/ocf_csgo/manifests/init.pp | 47 --------------- modules/ocf_ldap/files/munin/slapd-open-files | 18 ------ modules/ocf_ldap/manifests/init.pp | 5 -- modules/ocf_mail/files/site_ocf/aliases | 1 - modules/ocf_mail/manifests/logging.pp | 5 -- modules/ocf_ns/files/ping-report | 58 ------------------- modules/ocf_ns/manifests/init.pp | 4 -- .../files/rules.d/node.rules.yaml | 1 - modules/ocf_www/manifests/site/shorturl.pp | 1 - 22 files changed, 338 deletions(-) delete mode 100644 hieradata/nodes/jaws.yaml delete mode 100644 modules/ocf/files/munin/munin-node.conf delete mode 100644 modules/ocf/manifests/munin/node.pp delete mode 100644 modules/ocf/manifests/munin/plugin.pp delete mode 100755 modules/ocf_csgo/files/bin/update-csgo delete mode 100644 modules/ocf_csgo/files/etc/csgo-update.cmd delete mode 100755 modules/ocf_csgo/files/munin delete mode 100644 modules/ocf_csgo/manifests/init.pp delete mode 100755 modules/ocf_ldap/files/munin/slapd-open-files delete mode 100755 modules/ocf_ns/files/ping-report diff --git a/hieradata/nodes/dataloss.yaml b/hieradata/nodes/dataloss.yaml index 50514f6ce..7c122187c 100644 --- a/hieradata/nodes/dataloss.yaml +++ b/hieradata/nodes/dataloss.yaml @@ -5,7 +5,6 @@ ocf::packages::ntp::master: true ocf::packages::ntp::peers: - hal.ocf.berkeley.edu - pandemic.ocf.berkeley.edu - - jaws.ocf.berkeley.edu - riptide.ocf.berkeley.edu ocf_filehost::storage_device: '/dev/md/nfs' diff --git a/hieradata/nodes/hal.yaml b/hieradata/nodes/hal.yaml index 6a1e70715..7e0665ce4 100644 --- a/hieradata/nodes/hal.yaml +++ b/hieradata/nodes/hal.yaml @@ -7,7 +7,6 @@ ocf::networking::bond: true ocf::packages::ntp::master: true ocf::packages::ntp::peers: - - jaws.ocf.berkeley.edu - pandemic.ocf.berkeley.edu - riptide.ocf.berkeley.edu - dataloss.ocf.berkeley.edu diff --git a/hieradata/nodes/jaws.yaml b/hieradata/nodes/jaws.yaml deleted file mode 100644 index 4da081f6d..000000000 --- a/hieradata/nodes/jaws.yaml +++ /dev/null @@ -1,12 +0,0 @@ -classes: - - ocf_kvm - - ocf_kube::controller - -ocf::networking::bridge: true -ocf::networking::bond: true - -ocf::packages::ntp::master: true -ocf::packages::ntp::peers: - - hal.ocf.berkeley.edu - - pandemic.ocf.berkeley.edu - - riptide.ocf.berkeley.edu diff --git a/hieradata/nodes/pandemic.yaml b/hieradata/nodes/pandemic.yaml index b07ee969d..a033df96b 100644 --- a/hieradata/nodes/pandemic.yaml +++ b/hieradata/nodes/pandemic.yaml @@ -8,6 +8,5 @@ ocf::networking::bond: true ocf::packages::ntp::master: true ocf::packages::ntp::peers: - hal.ocf.berkeley.edu - - jaws.ocf.berkeley.edu - riptide.ocf.berkeley.edu - dataloss.ocf.berkeley.edu diff --git a/hieradata/nodes/riptide.yaml b/hieradata/nodes/riptide.yaml index 3f9a8853c..e2260d21e 100644 --- a/hieradata/nodes/riptide.yaml +++ b/hieradata/nodes/riptide.yaml @@ -9,5 +9,4 @@ ocf::packages::ntp::master: true ocf::packages::ntp::peers: - hal.ocf.berkeley.edu - pandemic.ocf.berkeley.edu - - jaws.ocf.berkeley.edu - dataloss.ocf.berkeley.edu diff --git a/modules/ocf/files/munin/munin-node.conf b/modules/ocf/files/munin/munin-node.conf deleted file mode 100644 index f42d214cb..000000000 --- a/modules/ocf/files/munin/munin-node.conf +++ /dev/null @@ -1,58 +0,0 @@ -# -# Example config-file for munin-node -# - -log_level 4 -log_file /var/log/munin/munin-node.log -pid_file /var/run/munin/munin-node.pid - -background 1 -setsid 1 - -user root -group root - - -# Regexps for files to ignore -ignore_file [\#~]$ -ignore_file DEADJOE$ -ignore_file \.bak$ -ignore_file %$ -ignore_file \.dpkg-(tmp|new|old|dist)$ -ignore_file \.rpm(save|new)$ -ignore_file \.pod$ - -# Set this if the client doesn't report the correct hostname when -# telnetting to localhost, port 4949 -# -#host_name localhost.localdomain - -# A list of addresses that are allowed to connect. This must be a -# regular expression, since Net::Server does not understand CIDR-style -# network notation unless the perl module Net::CIDR is installed. You -# may repeat the allow line as many times as you'd like - -allow ^169\.229\.226\.33$ -allow ^2607:f140:8801::1:33$ -allow ^169\.229\.226\.24$ -allow ^2607:f140:8801::1:24$ -allow ^127\.0\.0\.1$ -allow ^::1$ - -# If you have installed the Net::CIDR perl module, you can use one or more -# cidr_allow and cidr_deny address/mask patterns. A connecting client must -# match any cidr_allow, and not match any cidr_deny. Note that a netmask -# *must* be provided, even if it's /32 -# -# Example: -# -# cidr_allow 127.0.0.1/32 -# cidr_allow 192.0.2.0/24 -# cidr_deny 192.0.2.42/32 - -# Which address to bind to; -host * -# host 127.0.0.1 - -# And which port -port 4949 diff --git a/modules/ocf/manifests/init.pp b/modules/ocf/manifests/init.pp index 1a2523697..2856065df 100644 --- a/modules/ocf/manifests/init.pp +++ b/modules/ocf/manifests/init.pp @@ -13,7 +13,6 @@ include ocf::locale include ocf::logging include ocf::motd - include ocf::munin::node include ocf::networking include ocf::node_exporter include ocf::packages diff --git a/modules/ocf/manifests/munin/node.pp b/modules/ocf/manifests/munin/node.pp deleted file mode 100644 index aa188c19f..000000000 --- a/modules/ocf/manifests/munin/node.pp +++ /dev/null @@ -1,25 +0,0 @@ -# munin node config -class ocf::munin::node { - package { - ['munin-node', 'munin-plugins-core', 'munin-plugins-extra', - 'munin-libvirt-plugins']:; - } - - service { 'munin-node': - require => Package['munin-node']; - } - - file { '/etc/munin/munin-node.conf': - source => 'puppet:///modules/ocf/munin/munin-node.conf', - mode => '0644', - notify => Service['munin-node'], - require => Package['munin-node']; - } - - file { '/etc/munin/plugin-conf.d/ocf-plugin-conf': - content => template('ocf/munin/ocf-plugin-conf.erb'), - mode => '0644', - notify => Service['munin-node'], - require => Package['munin-node']; - } -} diff --git a/modules/ocf/manifests/munin/plugin.pp b/modules/ocf/manifests/munin/plugin.pp deleted file mode 100644 index be46d6d40..000000000 --- a/modules/ocf/manifests/munin/plugin.pp +++ /dev/null @@ -1,38 +0,0 @@ -# Munin plugin resource -# -# Can be used to produce custom graphs in Munin. The config should be applied -# at the *node* level, not on the master. -# -# Example usage: -# ocf::munin::plugin { 'csgo': -# source => 'puppet:///modules/ocf_srcds/munin'; -# } -# -# See for instructions on writing new plugins: -# http://munin-monitoring.org/wiki/HowToWritePlugins -define ocf::munin::plugin($source, $user = undef) { - $file_defaults = { - notify => Service['munin-node'], - require => Package['munin-node'], - } - - file { - "/etc/munin/plugins/${title}": - source => $source, - mode => '0755', - * => $file_defaults; - } - - if $user != undef { - file { "/etc/munin/plugin-conf.d/plugin-${title}": - ensure => present, - content => "[${title}]\nuser ${user}\n", - * => $file_defaults; - } - } else { - file { "/etc/munin/plugin-conf.d/plugin-${title}": - ensure => absent, - * => $file_defaults; - } - } -} diff --git a/modules/ocf_backups/files/rsnapshot.conf b/modules/ocf_backups/files/rsnapshot.conf index 23739a2c6..f37a6e42b 100644 --- a/modules/ocf_backups/files/rsnapshot.conf +++ b/modules/ocf_backups/files/rsnapshot.conf @@ -53,7 +53,6 @@ backup_script /opt/share/backups/backup-pgsql pgsql/ # remote servers backup ocfbackups@hal:/etc/libvirt/qemu/ servers/vm_xml/hal/ -backup ocfbackups@jaws:/etc/libvirt/qemu/ servers/vm_xml/jaws/ backup ocfbackups@pandemic:/etc/libvirt/qemu/ servers/vm_xml/pandemic/ backup ocfbackups@riptide:/etc/libvirt/qemu/ servers/vm_xml/riptide/ backup ocfbackups@scurvy:/etc/libvirt/qemu/ servers/vm_xml/scurvy/ @@ -69,8 +68,6 @@ backup ocfbackups@puppet:/opt/puppetlabs/ servers/puppet/ backup ocfbackups@puppetdb:/etc/puppetlabs/puppet/ssl/ servers/puppetdb/ -backup ocfbackups@munin:/var/lib/munin/ servers/munin/ - backup ocfbackups@apt:/opt/apt/ servers/apt/ backup ocfbackups@jenkins:/var/lib/jenkins/ servers/jenkins/ diff --git a/modules/ocf_csgo/files/bin/update-csgo b/modules/ocf_csgo/files/bin/update-csgo deleted file mode 100755 index 05088b15f..000000000 --- a/modules/ocf_csgo/files/bin/update-csgo +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -e -/opt/csgo/bin/steamcmd.sh +runscript /opt/csgo/etc/csgo-update.cmd diff --git a/modules/ocf_csgo/files/etc/csgo-update.cmd b/modules/ocf_csgo/files/etc/csgo-update.cmd deleted file mode 100644 index 6ce4c3a5b..000000000 --- a/modules/ocf_csgo/files/etc/csgo-update.cmd +++ /dev/null @@ -1,18 +0,0 @@ -// steamcmd command script to update the cs:go server -// -// run with: -// ./steamcmd.sh +runscript /opt/csgo/etc/csgo-update.cmd -// a full run takes about ~2 minutes (even if no updates exist) -// -// see https://developer.valvesoftware.com/wiki/SteamCMD#Automating_SteamCMD - -@ShutdownOnFailedCommand 1 -@NoPromptForPassword 1 - -login anonymous - -// install or update cs:go -force_install_dir /opt/csgo/srcds -app_update 740 validate - -quit diff --git a/modules/ocf_csgo/files/munin b/modules/ocf_csgo/files/munin deleted file mode 100755 index d65a84984..000000000 --- a/modules/ocf_csgo/files/munin +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -# Munin plugin for reporting the number of players currently on the OCF CS:GO -# server. -import socket -import sys - -SERVER = ('csgo', 27015) - - -def get_num_players(server): - """Returns the number of players on a Source Engine server, excluding - bots.""" - - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - sock.connect(SERVER) - - # https://developer.valvesoftware.com/wiki/Server_queries#A2S_INFO - req = b'\xff\xff\xff\xffTSource Engine Query\x00' - sock.send(req) - - response = sock.recv(4096) - response = response[4:] # cut off header - response = response.split(b'\0', 4)[4] # cut off 4 strings - - players, max_players, bots = map(int, response[2:5]) - return players - bots - - -if __name__ == '__main__': - if len(sys.argv) == 2 and sys.argv[1] == 'config': - print('graph_title CS:GO players') - print('graph_vlabel players') - print('graph_scale no') - print('players.label players') - sys.exit(0) - - print('players.value {}'.format(get_num_players(SERVER))) diff --git a/modules/ocf_csgo/manifests/init.pp b/modules/ocf_csgo/manifests/init.pp deleted file mode 100644 index 537bf2691..000000000 --- a/modules/ocf_csgo/manifests/init.pp +++ /dev/null @@ -1,47 +0,0 @@ -class ocf_csgo { - include ocf::apt::i386 - include ocf::firewall::allow_desktops - - user { 'ocfcsgo': - comment => 'Counter-Strike Server', - home => '/opt/csgo', - groups => ['sys'], - shell => '/bin/false'; - } - - file { - default: - owner => ocfcsgo, - group => ocfcsgo; - - ['/opt/csgo', '/opt/csgo/bin', '/opt/csgo/etc']: - ensure => directory, - mode => '0755'; - - '/opt/csgo/bin/update-csgo': - source => 'puppet:///modules/ocf_csgo/bin/update-csgo', - mode => '0755'; - - '/opt/csgo/etc/csgo-update.cmd': - source => 'puppet:///modules/ocf_csgo/etc/csgo-update.cmd'; - } - - exec { - 'download-steamcmd': - command => 'curl http://media.steampowered.com/installer/steamcmd_linux.tar.gz | tar xzf - -C /opt/csgo/bin', - user => ocfcsgo, - creates => '/opt/csgo/bin/steamcmd.sh', - notify => Exec['update-csgo'], - require => File['/opt/csgo/bin']; - - 'update-csgo': - command => '/opt/csgo/bin/update-csgo', - user => ocfcsgo, - refreshonly => true, - require => [File['/opt/csgo/bin/update-csgo']]; - } - - ocf::munin::plugin { 'csgo': - source => 'puppet:///modules/ocf_csgo/munin'; - } -} diff --git a/modules/ocf_ldap/files/munin/slapd-open-files b/modules/ocf_ldap/files/munin/slapd-open-files deleted file mode 100755 index 472a34478..000000000 --- a/modules/ocf_ldap/files/munin/slapd-open-files +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -eu -# Munin plugin for reporting the number of open files by slapd. - -if [ $# -eq 1 ] && [ "$1" == "config" ]; then - echo 'graph_title slapd open files' - echo 'graph_vlabel open files' - echo 'graph_scale no' - echo 'open_files.label open_files' - exit 0 -fi - -pid=$(pidof slapd) - -if [ "$pid" -gt 0 ]; then - echo "open_files.value $(find "/proc/$pid/fd/" | wc -l)" -else - exit 1 -fi diff --git a/modules/ocf_ldap/manifests/init.pp b/modules/ocf_ldap/manifests/init.pp index 26d3c538a..eacf5c0ec 100644 --- a/modules/ocf_ldap/manifests/init.pp +++ b/modules/ocf_ldap/manifests/init.pp @@ -152,11 +152,6 @@ special => 'daily', } - ocf::munin::plugin { 'slapd-open-files': - source => 'puppet:///modules/ocf_ldap/munin/slapd-open-files', - user => root, - } - # firewall input rule, allow ldaps, port number 636 ocf::firewall::firewall46 { '101 allow ldaps': diff --git a/modules/ocf_mail/files/site_ocf/aliases b/modules/ocf_mail/files/site_ocf/aliases index 18607d51a..658b26273 100644 --- a/modules/ocf_mail/files/site_ocf/aliases +++ b/modules/ocf_mail/files/site_ocf/aliases @@ -12,7 +12,6 @@ ocfstats: root jenkins: root rancid: root rancid-ocf: root -munin: root # archive of outgoing mail for nomail'd users nomail: /var/mail/nomail/nomail diff --git a/modules/ocf_mail/manifests/logging.pp b/modules/ocf_mail/manifests/logging.pp index 0eae64d63..03f596085 100644 --- a/modules/ocf_mail/manifests/logging.pp +++ b/modules/ocf_mail/manifests/logging.pp @@ -10,9 +10,4 @@ ensure => file, source => 'puppet:///modules/ocf_mail/site_ocf/logrotate/nomail'; } - - ocf::munin::plugin { 'mails-past-hour': - source => 'puppet:///modules/ocf_mail/site_ocf/munin/mails-past-hour', - user => root, - } } diff --git a/modules/ocf_ns/files/ping-report b/modules/ocf_ns/files/ping-report deleted file mode 100755 index 566e34303..000000000 --- a/modules/ocf_ns/files/ping-report +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -set -euo pipefail - -HOSTS=" -blackhole.ocf.berkeley.edu -cory.eecs.berkeley.edu -google.com -hal.ocf.berkeley.edu -jaws.ocf.berkeley.edu -ocf-2.eac.berkeley.edu -osuosl.org -pandemic.ocf.berkeley.edu -sipb.mit.edu -speedtest.dallas.linode.com -speedtest.frankfurt.linode.com -speedtest.london.linode.com -speedtest.newark.linode.com -speedtest.singapore.linode.com -speedtest.tokyo.linode.com -vlan635.inr-350-reccev.berkeley.edu -" - - -latency_to_host() { - local host="$1" - - # mean latency from 5 pings - ping -nq -i 0.2 -W3 -c5 "$host" | - awk '/^rtt / {print $4}' | - awk -F/ '{print $2}' -} - -clean() { - sed 's/[^a-z0-9]/_/g' <<< "$1" -} - - -if [ "$#" -ne 0 ]; then - if [ "$1" == 'config' ]; then - echo 'graph_title round-trip latency' - echo 'graph_vlabel latency (ms)' - echo 'graph_scale no' - for host in $HOSTS; do - echo "$(clean "$host").label $host" - done - else - echo "usage: $0 {config}" >&2 - exit 1 - fi -else - for host in $HOSTS; do - if latency=$(latency_to_host "$host"); then - echo "$(clean "$host").value $latency" - else - echo "fail: $host" >&2 - fi - done -fi diff --git a/modules/ocf_ns/manifests/init.pp b/modules/ocf_ns/manifests/init.pp index a344bf9cd..bf969fa8b 100644 --- a/modules/ocf_ns/manifests/init.pp +++ b/modules/ocf_ns/manifests/init.pp @@ -32,10 +32,6 @@ notify => Service['bind9']; } - ocf::munin::plugin { 'ping-report': - source => 'puppet:///modules/ocf_ns/ping-report', - } - ocf::firewall::firewall46 { '101 allow domain': opts => { diff --git a/modules/ocf_prometheus/files/rules.d/node.rules.yaml b/modules/ocf_prometheus/files/rules.d/node.rules.yaml index 38a564528..3a4960123 100644 --- a/modules/ocf_prometheus/files/rules.d/node.rules.yaml +++ b/modules/ocf_prometheus/files/rules.d/node.rules.yaml @@ -1,5 +1,4 @@ # Alerts for node metrics -# TODO: port Munin alerts to this groups: - name: node rules: diff --git a/modules/ocf_www/manifests/site/shorturl.pp b/modules/ocf_www/manifests/site/shorturl.pp index d76fa8961..43c63a7f1 100644 --- a/modules/ocf_www/manifests/site/shorturl.pp +++ b/modules/ocf_www/manifests/site/shorturl.pp @@ -102,7 +102,6 @@ {rewrite_rule => '^/minutes(/.*)?$ https://www.ocf.berkeley.edu/~staff/bod$1 [R]'}, {rewrite_rule => '^/mirrorstats$ https://grafana.ocf.berkeley.edu/d/Jo_bRsyiz/mirrors?orgId=1 [R]'}, {rewrite_rule => '^/mlk$ https://www.ocf.berkeley.edu/mlk [R]'}, - {rewrite_rule => '^/(mon|munin)$ https://munin.ocf.berkeley.edu/ [R]'}, {rewrite_rule => '^/mysql$ https://www.ocf.berkeley.edu/docs/services/mysql/ [R]'}, {rewrite_rule => '^/newstaff$ https://forms.gle/guESY2ykNkshNxsf8 [R]'}, {rewrite_rule => '^/notes$ https://notes.ocf.berkeley.edu/ [R]'}, From 8f429c69e569ff99bab0dcb93046af026c3fb260 Mon Sep 17 00:00:00 2001 From: kpengboy Date: Mon, 24 Oct 2022 18:10:42 -0700 Subject: [PATCH 10/12] ocf_desktop: remove pre-bullseye package code (#1159) (#1333) --- modules/ocf_desktop/manifests/packages.pp | 66 +++-------------------- 1 file changed, 8 insertions(+), 58 deletions(-) diff --git a/modules/ocf_desktop/manifests/packages.pp b/modules/ocf_desktop/manifests/packages.pp index 80b1394db..04bbb0018 100644 --- a/modules/ocf_desktop/manifests/packages.pp +++ b/modules/ocf_desktop/manifests/packages.pp @@ -20,13 +20,16 @@ 'gnome-calculator', 'gparted', 'hexchat', 'imagej', 'inkscape', 'lyx', 'musescore3', 'mpv', 'mssh', 'mumble', 'numlockx', 'simple-scan', 'ssh-askpass-gnome', 'texmaker', - 'texstudio', 'vlc', 'xarchiver', 'xcape', 'xournal', 'xterm']:; + 'texstudio', 'tigervnc-viewer', 'vlc', 'xarchiver', 'xcape', 'xournal', + 'xterm']:; # desktop ['desktop-base', 'anacron', 'accountsservice', 'arc-theme', 'desktop-file-utils', 'gnome-icon-theme', 'paper-icon-theme', 'redshift', 'xfce4-whiskermenu-plugin']:; # desktop helpers ['libimage-exiftool-perl']:; + # development: + ['openjdk-17-jdk']:; # display manager ['lightdm', 'lightdm-gtk-greeter', 'libpam-trimspaces']:; # games @@ -46,63 +49,10 @@ ['wakeonlan']:; # Xorg ['xclip', 'xdotool', 'xsel', 'xserver-xorg', 'xscreensaver', 'freerdp2-x11']:; - } - - if $::lsbdistcodename == 'stretch' { - package { - [ - # preload hasn't been updated since 2009, and I'm not sure we really - # get anything out of it in terms of performance improvements at this - # point anyway. - 'preload', - - # Zenmap depends on Python 2 and is therefore no longer in bullseye - 'zenmap', - - # FUSE and exfat - 'fuse', - 'exfat-fuse', - - # Florence was removed from bullseye due to deprecated dependency - # We should find an alternative - # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=947521 - 'florence', - ]:; - } - } - if $::lsbdistcodename == 'buster' { - package { - [ - # Zenmap depends on Python 2 and is therefore no longer in bullseye - 'zenmap', - - # FUSE and exfat - 'fuse', - 'exfat-fuse', - - # Florence was removed from bullseye due to deprecated dependency - # We should find an alternative - # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=947521 - 'florence', - ]:; - } - } - if $::lsbdistcodename == 'bullseye' { - package { - [ - # OpenJDK 17 (LTS) is in bullseye - 'openjdk-17-jdk', - - # Matchbox is what we use on our RPi - 'matchbox-keyboard', - - # x4vncviewer is no longer present - 'tigervnc-viewer', - - # sshfs depends on fuse3 on bullseye - 'fuse3', - ]:; - } + # Matchbox is what we use on our RPi + ['matchbox-keyboard']:; + # sshfs depends on fuse3 on bullseye + ['fuse3']:; } # Remove some packages From 73519ee88d17a163510b0188879679ceeea4733a Mon Sep 17 00:00:00 2001 From: Open Computing Facility Date: Mon, 24 Oct 2022 18:11:03 -0700 Subject: [PATCH 11/12] automatically bump version to 1.4.0 (#1330) --- hieradata/common.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hieradata/common.yaml b/hieradata/common.yaml index 125e76ed9..6a1eeafe0 100644 --- a/hieradata/common.yaml +++ b/hieradata/common.yaml @@ -111,5 +111,5 @@ kube_dev::controller_nodes: - hozer-74 # Prometheus config -prometheus::node_exporter::version: 1.3.1 +prometheus::node_exporter::version: 1.4.0 prometheus::node_exporter::extra_options: '--collector.textfile.directory /srv/prometheus' From d148bfea422c795e7d08d598358f80c0fb91b8e1 Mon Sep 17 00:00:00 2001 From: Jonathan Zhang Date: Mon, 24 Oct 2022 21:22:21 -0400 Subject: [PATCH 12/12] Bring puppet up to date with current fallingrocks configuration (#1276) * fix: bring puppet up-to-date with fallingrocks * fix: bring puppet up-to-date with fallingrocks * fix: indent * fix: eof * fix: enable sendfile * fix: nginx package * feat: add linux mint repos * feat: add openwrt * fix: add rsync module * fix: project file location * fix: update * fix: update * fix: update * fix: update * fix: update * fix: update * fix: remove centos-debuginfo * feat: add monitoring for linuxmint-packages * feat: add monitoring for raspi * fix: fix monitoring for linuxmint-packages * style: fix indent * feat: http header health check * feat: add rocky health check * fix: use official FreeBSD mirror * fix: use a faster FreeBSD mirror * feat: reintroduce fancyindex * feat: reintroduce fancyindex * fix: don't show header on every page * feat: better fancyindex * fix: fancyindex * misc: donation push?? * misc: optimization * fix: css * fix: css * feat: incl mx linux * feat: new look * fix: fix duplicate statements * feat: add cran, almalinux * fix: switch upstream for fedora * fix: apparently they added our new ips (raspbian) * fix: wrong rsync module * feat: add osdn * fix: fix alpine, osdn * fix: add osdn * fix: add osdn * fix: add rsync entry for videolan-ftp * feat: add libreelec and blackarch * fix: puppet syntax * fix: puppet syntax * feat: opensuse tumbleweed * fix: switch to a different module * fix: block weird centos iso requests * fix: actually block weird centos iso requests * fix: sync puppet files for fallingrocks * fix: new GIMP rsync endpoint * feat: add pikvm repos * fix: syntax --- hieradata/dummy_secrets.yaml | 4 ++ modules/ocf_apt/manifests/init.pp | 18 ++--- modules/ocf_mirrors/files/FOOTER.html | 2 +- modules/ocf_mirrors/files/README.html | 14 ++-- .../ocf_mirrors/files/collect-mirrors-stats | 3 +- modules/ocf_mirrors/files/healthcheck | 15 +++- .../files/project/almalinux/sync-archive | 4 ++ .../files/project/blackarch/sync-archive | 3 + .../files/project/blender/sync-archive | 4 +- .../project/centos-debuginfo/sync-archive | 2 - .../files/project/cran/sync-archive | 3 + .../files/project/freebsd/sync-archive | 2 +- .../files/project/gimp/sync-archive | 4 ++ .../files/project/ipfire/sync-archive | 3 + .../files/project/libreelec/sync-archive | 3 + .../files/project/linux-mint/sync-archive | 3 + .../project/linuxmint-packages/sync-archive | 4 ++ .../files/project/mx-linux/sync-archive | 4 ++ .../files/project/mx-packages/sync-archive | 4 ++ .../files/project/opensuse/sync-archive | 4 +- .../files/project/openwrt/sync-archive | 2 + .../files/project/osdn/sync-archive | 4 ++ .../files/project/pikvm/sync-archive | 3 + .../files/project/rpmfusion/sync-archive | 4 ++ .../files/project/ubuntu/sync-releases | 2 +- modules/ocf_mirrors/files/rsyncd.conf | 72 +++++++++++++++++-- modules/ocf_mirrors/manifests/init.pp | 52 ++++++++------ .../manifests/projects/almalinux.pp | 17 +++++ .../manifests/projects/blackarch.pp | 25 +++++++ .../ocf_mirrors/manifests/projects/blender.pp | 9 ++- .../ocf_mirrors/manifests/projects/cran.pp | 17 +++++ .../ocf_mirrors/manifests/projects/fedora.pp | 23 +++++- .../ocf_mirrors/manifests/projects/gimp.pp | 28 ++++++++ .../ocf_mirrors/manifests/projects/ipfire.pp | 19 +++++ .../manifests/projects/libreelec.pp | 18 +++++ .../manifests/projects/linux_mint.pp | 19 +++++ .../manifests/projects/linuxmint_packages.pp | 27 +++++++ .../manifests/projects/mx_linux.pp | 26 +++++++ .../manifests/projects/mx_packages.pp | 26 +++++++ .../ocf_mirrors/manifests/projects/openwrt.pp | 19 +++++ .../ocf_mirrors/manifests/projects/osdn.pp | 19 +++++ .../ocf_mirrors/manifests/projects/pikvm.pp | 17 +++++ .../manifests/projects/raspbian.pp | 2 +- .../ocf_mirrors/manifests/projects/raspi.pp | 8 +++ .../ocf_mirrors/manifests/projects/rocky.pp | 7 ++ .../manifests/projects/rpmfusion.pp | 19 +++++ 46 files changed, 528 insertions(+), 59 deletions(-) create mode 100755 modules/ocf_mirrors/files/project/almalinux/sync-archive create mode 100755 modules/ocf_mirrors/files/project/blackarch/sync-archive delete mode 100755 modules/ocf_mirrors/files/project/centos-debuginfo/sync-archive create mode 100755 modules/ocf_mirrors/files/project/cran/sync-archive create mode 100644 modules/ocf_mirrors/files/project/gimp/sync-archive create mode 100644 modules/ocf_mirrors/files/project/ipfire/sync-archive create mode 100755 modules/ocf_mirrors/files/project/libreelec/sync-archive create mode 100644 modules/ocf_mirrors/files/project/linux-mint/sync-archive create mode 100644 modules/ocf_mirrors/files/project/linuxmint-packages/sync-archive create mode 100644 modules/ocf_mirrors/files/project/mx-linux/sync-archive create mode 100644 modules/ocf_mirrors/files/project/mx-packages/sync-archive create mode 100644 modules/ocf_mirrors/files/project/openwrt/sync-archive create mode 100755 modules/ocf_mirrors/files/project/osdn/sync-archive create mode 100755 modules/ocf_mirrors/files/project/pikvm/sync-archive create mode 100755 modules/ocf_mirrors/files/project/rpmfusion/sync-archive create mode 100644 modules/ocf_mirrors/manifests/projects/almalinux.pp create mode 100644 modules/ocf_mirrors/manifests/projects/blackarch.pp create mode 100644 modules/ocf_mirrors/manifests/projects/cran.pp create mode 100644 modules/ocf_mirrors/manifests/projects/gimp.pp create mode 100644 modules/ocf_mirrors/manifests/projects/ipfire.pp create mode 100644 modules/ocf_mirrors/manifests/projects/libreelec.pp create mode 100644 modules/ocf_mirrors/manifests/projects/linux_mint.pp create mode 100644 modules/ocf_mirrors/manifests/projects/linuxmint_packages.pp create mode 100644 modules/ocf_mirrors/manifests/projects/mx_linux.pp create mode 100644 modules/ocf_mirrors/manifests/projects/mx_packages.pp create mode 100644 modules/ocf_mirrors/manifests/projects/openwrt.pp create mode 100644 modules/ocf_mirrors/manifests/projects/osdn.pp create mode 100644 modules/ocf_mirrors/manifests/projects/pikvm.pp create mode 100644 modules/ocf_mirrors/manifests/projects/rpmfusion.pp diff --git a/hieradata/dummy_secrets.yaml b/hieradata/dummy_secrets.yaml index 77178ecc5..8ae8406cf 100644 --- a/hieradata/dummy_secrets.yaml +++ b/hieradata/dummy_secrets.yaml @@ -57,8 +57,12 @@ kubernetes::keepalived::secret: dummysecret prometheus::docker_metrics_password: dummypassword mirrors::archlinuxcn_sync_password: dummypassword +mirrors::blender_sync_password: dummypassword mirrors::finnix_sync_password: dummypassword +mirrors::gimp_sync_password: dummypassword mirrors::gnome_sync_password: dummypassword +mirrors::mx_linux_sync_password: dummypassword +mirrors::mx_packages_sync_password: dummypassword xmpp::root_password: dummypassword xmpp::prosody_mysql_password: dummypassword diff --git a/modules/ocf_apt/manifests/init.pp b/modules/ocf_apt/manifests/init.pp index 6aa058a11..a47ab833f 100644 --- a/modules/ocf_apt/manifests/init.pp +++ b/modules/ocf_apt/manifests/init.pp @@ -10,8 +10,6 @@ package { [ - 'nginx-full', - 'libnginx-mod-http-fancyindex', 'reprepro', ]:; } @@ -84,19 +82,13 @@ ipv6_enable => true, ipv6_listen_port => 80, format_log => 'main', - raw_append => @(END), - fancyindex on; - fancyindex_exact_size off; - END + autoindex => 'on', } nginx::resource::location { '= /': - ensure => present, - server => ['apt.ocf.berkeley.edu', 'apt'], - www_root => '/opt/apt/ftp', - ssl => true, - raw_append => @(END), - fancyindex_header README.html; - END + ensure => present, + server => ['apt.ocf.berkeley.edu', 'apt'], + www_root => '/opt/apt/ftp', + ssl => true, } nginx::resource::location { '~ /\.(?!well-known).*': ensure => present, diff --git a/modules/ocf_mirrors/files/FOOTER.html b/modules/ocf_mirrors/files/FOOTER.html index 97f5701dd..a781b550b 100644 --- a/modules/ocf_mirrors/files/FOOTER.html +++ b/modules/ocf_mirrors/files/FOOTER.html @@ -1 +1 @@ -

Hosted by the Open Computing Facility at UC Berkeley.

+

Hosted by the Open Computing Facility at UC Berkeley. We appreciate your donation.

diff --git a/modules/ocf_mirrors/files/README.html b/modules/ocf_mirrors/files/README.html index 265064171..9dc00aabc 100644 --- a/modules/ocf_mirrors/files/README.html +++ b/modules/ocf_mirrors/files/README.html @@ -39,20 +39,20 @@ .ocf-penguin-swing { position: absolute; - height: 326px; + height: 200%; left: 50%; right: 0; top: -95px; - opacity: 0.33; + opacity: 0.07; background-image: url("//static.ocf.berkeley.edu/img/penguin-swing.svg"); background-repeat: no-repeat; - background-size: 293px 326px; + background-size: 100%; pointer-events: none } OCF Mirrors -

Open Computing Facility Mirrors

+

🐧 Open Computing Facility Mirrors

The Open Computing Facility at @@ -81,3 +81,9 @@

Open Computing Facility Mirrors

To request for another project to be mirrored or to report a problem, send us an email.

+

+ The OCF is completely run by students. If you find this mirror helpful, please donate to the OCF. Thank you! +

+

+ New: Some statistics. +

diff --git a/modules/ocf_mirrors/files/collect-mirrors-stats b/modules/ocf_mirrors/files/collect-mirrors-stats index 3b6adcb7a..d1e4f491b 100755 --- a/modules/ocf_mirrors/files/collect-mirrors-stats +++ b/modules/ocf_mirrors/files/collect-mirrors-stats @@ -20,7 +20,8 @@ MIRRORS_DATA_PATH = Path('/opt/mirrors/ftp') NGINX_LOG_PATH = Path('/var/log/nginx') NGINX_LOG_FILES = [ Path('mirrors.ocf.berkeley.edu.access.log'), - Path('ssl-mirrors.ocf.berkeley.edu.access.log') + Path('ssl-mirrors.ocf.berkeley.edu.access.log'), + Path('ca.us.mirror.archlinuxarm.org.access.log') ] RSYNC_LOG_PATH = Path('/var/log/rsync') RSYNC_LOG_REGEX = re.compile( diff --git a/modules/ocf_mirrors/files/healthcheck b/modules/ocf_mirrors/files/healthcheck index aedad5c07..ab7176566 100755 --- a/modules/ocf_mirrors/files/healthcheck +++ b/modules/ocf_mirrors/files/healthcheck @@ -34,6 +34,8 @@ def update_func(healthcheck): return get_updated_datetime elif healthcheck == 'recursive_ls': return get_updated_recursive_ls + elif healthcheck == 'http_last_modified': + return get_updated_http_last_modified else: raise ValueError('Unsupported type: {}'.format(healthcheck)) @@ -46,7 +48,7 @@ def get_updated_datetime(mirror_url): """ req = requests.get(mirror_url) req.raise_for_status() - return dateutil.parser.parse(req.text.splitlines()[0]) + return dateutil.parser.parse(req.text.splitlines()[0], fuzzy=True) def get_updated_unix_timestamp(mirror_url): @@ -75,6 +77,17 @@ def get_updated_recursive_ls(mirror_url): return dateutil.parser.parse(max(dates)) +def get_updated_http_last_modified(mirror_url): + """Find the time the host was last synced. + The header content should be of the form "Sat, 30 Apr 2022 19:16:24 GMT". + >>> get_updated_http_last_modified(mirror_url) + datetime.datetime(2022, 4, 30, 19, 16, 24, tzinfo=tzutc()) + """ + req = requests.head(mirror_url) + req.raise_for_status() + return dateutil.parser.parse(req.headers['last-modified']) + + def get_updated_debian(mirror_url): """Find the time the host was last synced. The page of should have a line of the form "Date: Fri, 19 Jan 2018 19:26:41 UTC" diff --git a/modules/ocf_mirrors/files/project/almalinux/sync-archive b/modules/ocf_mirrors/files/project/almalinux/sync-archive new file mode 100755 index 000000000..df9c3642e --- /dev/null +++ b/modules/ocf_mirrors/files/project/almalinux/sync-archive @@ -0,0 +1,4 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -avSH -f 'R .~tmp~' --delete-delay --delay-updates \ + rsync://rsync.repo.almalinux.org/almalinux/ \ + /opt/mirrors/ftp/almalinux diff --git a/modules/ocf_mirrors/files/project/blackarch/sync-archive b/modules/ocf_mirrors/files/project/blackarch/sync-archive new file mode 100755 index 000000000..c5c660efe --- /dev/null +++ b/modules/ocf_mirrors/files/project/blackarch/sync-archive @@ -0,0 +1,3 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -auvzH --delete --delete-after --delay-updates --safe-links \ + rsync://blackarch.org/blackarch/ /opt/mirrors/ftp/blackarch diff --git a/modules/ocf_mirrors/files/project/blender/sync-archive b/modules/ocf_mirrors/files/project/blender/sync-archive index b6f542fea..ff4dc4c03 100755 --- a/modules/ocf_mirrors/files/project/blender/sync-archive +++ b/modules/ocf_mirrors/files/project/blender/sync-archive @@ -1,3 +1,3 @@ #!/bin/sh -eu -/usr/local/bin/rsync-no-vanished -azH --delete --delete-after --delay-updates --safe-links \ - rsync://mirror.clarkson.edu/blender/ /opt/mirrors/ftp/blender +/usr/local/bin/rsync-no-vanished -avzH --delete --delete-after --delay-updates --safe-links --password-file /opt/mirrors/project/blender/sync_password \ + ocfberkeley@download.blender.org::blender /opt/mirrors/ftp/blender diff --git a/modules/ocf_mirrors/files/project/centos-debuginfo/sync-archive b/modules/ocf_mirrors/files/project/centos-debuginfo/sync-archive deleted file mode 100755 index e8f221aaf..000000000 --- a/modules/ocf_mirrors/files/project/centos-debuginfo/sync-archive +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -eu -/usr/local/bin/rsync-no-vanished -aqzH --delete --exclude "4" --exclude "5" mirror.facebook.net::centos-debuginfo /opt/mirrors/ftp/centos-debuginfo diff --git a/modules/ocf_mirrors/files/project/cran/sync-archive b/modules/ocf_mirrors/files/project/cran/sync-archive new file mode 100755 index 000000000..c56b499ca --- /dev/null +++ b/modules/ocf_mirrors/files/project/cran/sync-archive @@ -0,0 +1,3 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -rptlzv --delete cran.r-project.org::CRAN \ + /opt/mirrors/ftp/cran diff --git a/modules/ocf_mirrors/files/project/freebsd/sync-archive b/modules/ocf_mirrors/files/project/freebsd/sync-archive index fb21a82e7..183d11e52 100644 --- a/modules/ocf_mirrors/files/project/freebsd/sync-archive +++ b/modules/ocf_mirrors/files/project/freebsd/sync-archive @@ -1,3 +1,3 @@ #!/bin/sh -eu /usr/local/bin/rsync-no-vanished -avzH --safe-links --delete-after --delay-updates --no-motd \ - rsync://mirror.csclub.uwaterloo.ca/FreeBSD /opt/mirrors/ftp/freebsd + rsync://ftp.dk.FreeBSD.org/FreeBSD /opt/mirrors/ftp/freebsd diff --git a/modules/ocf_mirrors/files/project/gimp/sync-archive b/modules/ocf_mirrors/files/project/gimp/sync-archive new file mode 100644 index 000000000..94416a1de --- /dev/null +++ b/modules/ocf_mirrors/files/project/gimp/sync-archive @@ -0,0 +1,4 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -aPvh --delete --delete-after --delay-updates \ + --safe-links --password-file=/opt/mirrors/project/gimp/sync_password \ + ocf-berkeley-edu@master.gnome.org::gimp /opt/mirrors/ftp/gimp diff --git a/modules/ocf_mirrors/files/project/ipfire/sync-archive b/modules/ocf_mirrors/files/project/ipfire/sync-archive new file mode 100644 index 000000000..b875e531a --- /dev/null +++ b/modules/ocf_mirrors/files/project/ipfire/sync-archive @@ -0,0 +1,3 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -avHz --delete --delay-updates \ + rsync.ipfire.org::current /opt/mirrors/ftp/ipfire/ diff --git a/modules/ocf_mirrors/files/project/libreelec/sync-archive b/modules/ocf_mirrors/files/project/libreelec/sync-archive new file mode 100755 index 000000000..2bbfade81 --- /dev/null +++ b/modules/ocf_mirrors/files/project/libreelec/sync-archive @@ -0,0 +1,3 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -auvzH --delete --delete-after --delay-updates --safe-links \ + rsync://releases.libreelec.tv/releases/ /opt/mirrors/ftp/libreelec diff --git a/modules/ocf_mirrors/files/project/linux-mint/sync-archive b/modules/ocf_mirrors/files/project/linux-mint/sync-archive new file mode 100644 index 000000000..9a2bbd8b5 --- /dev/null +++ b/modules/ocf_mirrors/files/project/linux-mint/sync-archive @@ -0,0 +1,3 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -avHP --delete-after pub.linuxmint.com::pub \ + /opt/mirrors/ftp/linux-mint diff --git a/modules/ocf_mirrors/files/project/linuxmint-packages/sync-archive b/modules/ocf_mirrors/files/project/linuxmint-packages/sync-archive new file mode 100644 index 000000000..0fc83869c --- /dev/null +++ b/modules/ocf_mirrors/files/project/linuxmint-packages/sync-archive @@ -0,0 +1,4 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -avHP --delete-after \ + rsync-packages.linuxmint.com::packages \ + /opt/mirrors/ftp/linuxmint-packages diff --git a/modules/ocf_mirrors/files/project/mx-linux/sync-archive b/modules/ocf_mirrors/files/project/mx-linux/sync-archive new file mode 100644 index 000000000..e0228e081 --- /dev/null +++ b/modules/ocf_mirrors/files/project/mx-linux/sync-archive @@ -0,0 +1,4 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -rtv --delete --delete-after --delay-updates \ + --safe-links --password-file=/opt/mirrors/project/mx-linux/sync_password \ + downstreamtestuser@rsync-mxlinux.org::MX-Linux/ /opt/mirrors/ftp/mx-linux diff --git a/modules/ocf_mirrors/files/project/mx-packages/sync-archive b/modules/ocf_mirrors/files/project/mx-packages/sync-archive new file mode 100644 index 000000000..3ab1c1a01 --- /dev/null +++ b/modules/ocf_mirrors/files/project/mx-packages/sync-archive @@ -0,0 +1,4 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -rtv --delete --delete-after --delay-updates \ + --safe-links --password-file=/opt/mirrors/project/mx-packages/sync_password \ + rsuser@iso.mxrepo.com::workspace /opt/mirrors/ftp/mx-packages diff --git a/modules/ocf_mirrors/files/project/opensuse/sync-archive b/modules/ocf_mirrors/files/project/opensuse/sync-archive index c67c38895..e39fa7d2d 100644 --- a/modules/ocf_mirrors/files/project/opensuse/sync-archive +++ b/modules/ocf_mirrors/files/project/opensuse/sync-archive @@ -1,3 +1,3 @@ #!/bin/sh -eu -/usr/local/bin/rsync-no-vanished -rlptyH --delay-updates --delete-delay -v \ - rsync://mirror.datto.com/opensuse /opt/mirrors/ftp/opensuse +/usr/local/bin/rsync-no-vanished -rlptyH --delay-updates --delete-delay --exclude debug/ -v \ + rsync://stage.opensuse.org/opensuse-full-really-everything/opensuse/ /opt/mirrors/ftp/opensuse diff --git a/modules/ocf_mirrors/files/project/openwrt/sync-archive b/modules/ocf_mirrors/files/project/openwrt/sync-archive new file mode 100644 index 000000000..cf818bea2 --- /dev/null +++ b/modules/ocf_mirrors/files/project/openwrt/sync-archive @@ -0,0 +1,2 @@ +#!/bin/sh -eu +rsync -avHP --delete rsync://downloads.openwrt.org/downloads/ /opt/mirrors/ftp/openwrt diff --git a/modules/ocf_mirrors/files/project/osdn/sync-archive b/modules/ocf_mirrors/files/project/osdn/sync-archive new file mode 100755 index 000000000..7352932d0 --- /dev/null +++ b/modules/ocf_mirrors/files/project/osdn/sync-archive @@ -0,0 +1,4 @@ +#!/bin/sh -eu +# compression disabled +/usr/local/bin/rsync-no-vanished -rplHtSPv --delete --delete-after \ + rsync://mirror.math.princeton.edu/pub/osdn/ /tank/osdn diff --git a/modules/ocf_mirrors/files/project/pikvm/sync-archive b/modules/ocf_mirrors/files/project/pikvm/sync-archive new file mode 100755 index 000000000..ea6c178a7 --- /dev/null +++ b/modules/ocf_mirrors/files/project/pikvm/sync-archive @@ -0,0 +1,3 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -auzHv --delete --delete-after --delay-updates --safe-links \ + rsync://files.pikvm.org/repos/ /opt/mirrors/ftp/pikvm diff --git a/modules/ocf_mirrors/files/project/rpmfusion/sync-archive b/modules/ocf_mirrors/files/project/rpmfusion/sync-archive new file mode 100755 index 000000000..63f27bc17 --- /dev/null +++ b/modules/ocf_mirrors/files/project/rpmfusion/sync-archive @@ -0,0 +1,4 @@ +#!/bin/sh -eu +/usr/local/bin/rsync-no-vanished -avHP \ + rsync://download1.rpmfusion.org/rpmfusion/ \ + /opt/mirrors/ftp/rpmfusion/ diff --git a/modules/ocf_mirrors/files/project/ubuntu/sync-releases b/modules/ocf_mirrors/files/project/ubuntu/sync-releases index e4e200025..369803d60 100755 --- a/modules/ocf_mirrors/files/project/ubuntu/sync-releases +++ b/modules/ocf_mirrors/files/project/ubuntu/sync-releases @@ -14,7 +14,7 @@ warn() { # Find a source mirror near you which supports rsync on # https://launchpad.net/ubuntu/+cdmirrors # rsync://.rsync.releases.ubuntu.com/releases should always work -RSYNCSOURCE=rsync://mirrors.mit.edu/ubuntu-releases +RSYNCSOURCE=rsync://cdimage.ubuntu.com/releases # Define where you want the mirror-data to be on your mirror BASEDIR=/opt/mirrors/ftp/ubuntu-releases/ diff --git a/modules/ocf_mirrors/files/rsyncd.conf b/modules/ocf_mirrors/files/rsyncd.conf index 57b767763..e941e540d 100644 --- a/modules/ocf_mirrors/files/rsyncd.conf +++ b/modules/ocf_mirrors/files/rsyncd.conf @@ -5,7 +5,7 @@ use chroot = yes read only = yes list = yes timeout = 60 -max connections = 60 +max connections = 200 socket options = SO_KEEPALIVE dont compress = *.gz *.tgz *.zip *.z *.rpm *.deb *.iso *.bz2 *.tbz *.zst *.xz reverse lookup = no @@ -15,6 +15,11 @@ reverse lookup = no path = /opt/mirrors/ftp/apache log file = /var/log/rsync/apache.log +[almalinux] + comment = almalinux mirror + path = /opt/mirrors/ftp/almalinux + log file = /var/log/rsync/almalinux.log + [alpine] comment = alpine mirror path = /opt/mirrors/ftp/alpine @@ -25,6 +30,11 @@ reverse lookup = no path = /opt/mirrors/ftp/archlinux log file = /var/log/rsync/archlinux.log +[archlinuxarm] + comment = archlinuxarm package mirror + path = /opt/mirrors/ftp/archlinuxarm + log file = /var/log/rsync/archlinuxarm.log + [archlinuxcn] comment = archlinuxcn package mirror path = /opt/mirrors/ftp/archlinuxcn @@ -35,6 +45,11 @@ reverse lookup = no path = /opt/mirrors/ftp/artix-linux log file = /var/log/rsync/artix-linux.log +[blackarch] + comment = blackarch package mirror + path = /opt/mirrors/ftp/blackarch + log file = /var/log/rsync/blackarch.log + [blender] comment = blender package mirror path = /opt/mirrors/ftp/blender @@ -50,16 +65,16 @@ reverse lookup = no path = /opt/mirrors/ftp/centos-altarch log file = /var/log/rsync/centos-altarch.log -[centos-debuginfo] - comment = centos debuginfo mirror - path = /opt/mirrors/ftp/centos-debuginfo - log file = /var/log/rsync/centos-debuginfo.log - [centos-stream] comment = centos stream mirror path = /opt/mirrors/ftp/centos-stream log file = /var/log/rsync/centos-stream.log +[cran] + comment = cran mirror + path = /opt/mirrors/ftp/cran + log file = /var/log/rsync/cran.log + [debian] comment = debian package mirror path = /opt/mirrors/ftp/debian @@ -110,6 +125,11 @@ reverse lookup = no path = /opt/mirrors/ftp/gentoo-portage log file = /var/log/rsync/gentoo-portage.log +[gimp] + comment = gimp project mirror + path = /opt/mirrors/ftp/gimp + log file = /var/log/rsync/gimp.log + [gnome] comment = gnome project mirror path = /opt/mirrors/ftp/gnome @@ -120,6 +140,11 @@ reverse lookup = no path = /opt/mirrors/ftp/gnu log file = /var/log/rsync/gnu.log +[ipfire] + comment = ipfire mirror + path = /opt/mirrors/ftp/ipfire + log file = /var/log/rsync/ipfire.log + [kali] comment = kali mirror path = /opt/mirrors/ftp/kali @@ -140,6 +165,21 @@ reverse lookup = no path = /opt/mirrors/ftp/kde-applicationdata log file = /var/log/rsync/kde-applicationdata.log +[libreelec] + comment = libreelec mirror + path = /opt/mirrors/ftp/libreelec + log file = /var/log/rsync/libreelec.log + +[linux-mint] + comment = linux mint mirror + path = /opt/mirrors/ftp/linux-mint + log file = /var/log/rsync/linux-mint.log + +[linuxmint-packages] + comment = linuxmint-packages mirror + path = /opt/mirrors/ftp/linuxmint-packages + log file = /var/log/rsync/linuxmint-packages.log + [manjaro] comment = manjaro package mirror path = /opt/mirrors/ftp/manjaro @@ -155,6 +195,16 @@ reverse lookup = no path = /opt/mirrors/ftp/opensuse log file = /var/log/rsync/opensuse.log +[openwrt] + comment = openwrt mirror + path = /opt/mirrors/ftp/openwrt + log file = /var/log/rsync/openwrt.log + +[osdn] + comment = osdn mirror + path = /opt/mirrors/ftp/osdn + log file = /var/log/rsync/osdn.log + [parrot] comment = parrot iso and archive mirror path = /opt/mirrors/ftp/parrot @@ -195,6 +245,11 @@ reverse lookup = no path = /opt/mirrors/ftp/rocky log file = /var/log/rsync/rocky.log +[rpmfusion] + comment = rpmfusion mirror + path = /opt/mirrors/ftp/rpmfusion + log file = /var/log/rsync/rpmfusion.log + [sage] comment = sagemath mirror path = /opt/mirrors/ftp/sage @@ -245,4 +300,9 @@ reverse lookup = no path = /opt/mirrors/ftp/ubuntu-ports-releases log file = /var/log/rsync/ubuntu-ports-releases.log +[videolan-ftp] + comment = videolan-ftp mirror + path = /opt/mirrors/ftp/videolan-ftp + log file = /var/log/rsync/videolan-ftp.log + # vim: ts=4 sts=4 sw=4 noet: diff --git a/modules/ocf_mirrors/manifests/init.pp b/modules/ocf_mirrors/manifests/init.pp index 5e742e9d5..bcc2f2a61 100644 --- a/modules/ocf_mirrors/manifests/init.pp +++ b/modules/ocf_mirrors/manifests/init.pp @@ -6,14 +6,17 @@ include ocf_mirrors::firewall_input # projects include ocf_mirrors::projects::apache + include ocf_mirrors::projects::almalinux include ocf_mirrors::projects::alpine include ocf_mirrors::projects::archlinux include ocf_mirrors::projects::archlinuxcn include ocf_mirrors::projects::artix_linux + include ocf_mirrors::projects::blackarch include ocf_mirrors::projects::blender include ocf_mirrors::projects::centos include ocf_mirrors::projects::centos_altarch include ocf_mirrors::projects::centos_stream + include ocf_mirrors::projects::cran include ocf_mirrors::projects::debian include ocf_mirrors::projects::debian_nonfree include ocf_mirrors::projects::devuan @@ -23,22 +26,33 @@ include ocf_mirrors::projects::freebsd include ocf_mirrors::projects::gentoo_distfiles include ocf_mirrors::projects::gentoo_portage + include ocf_mirrors::projects::gimp include ocf_mirrors::projects::gnome include ocf_mirrors::projects::gnu + include ocf_mirrors::projects::ipfire include ocf_mirrors::projects::kali include ocf_mirrors::projects::kde include ocf_mirrors::projects::kde_applicationdata + include ocf_mirrors::projects::libreelec + include ocf_mirrors::projects::linux_mint + include ocf_mirrors::projects::linuxmint_packages include ocf_mirrors::projects::manjaro + include ocf_mirrors::projects::mx_linux + include ocf_mirrors::projects::mx_packages include ocf_mirrors::projects::openbsd include ocf_mirrors::projects::opensuse + include ocf_mirrors::projects::openwrt + include ocf_mirrors::projects::osdn include ocf_mirrors::projects::parrot include ocf_mirrors::projects::parabola + include ocf_mirrors::projects::pikvm include ocf_mirrors::projects::puppetlabs include ocf_mirrors::projects::qt include ocf_mirrors::projects::qubes include ocf_mirrors::projects::raspbian include ocf_mirrors::projects::raspi include ocf_mirrors::projects::rocky + include ocf_mirrors::projects::rpmfusion include ocf_mirrors::projects::sage include ocf_mirrors::projects::slackware include ocf_mirrors::projects::tails @@ -75,9 +89,8 @@ '::nginx': manage_repo => false, include_modules_enabled => true, - sendfile => 'off', http_raw_append => @(END); - gzip on; + sendfile_max_chunk 20m; log_format main '$remote_addr - $remote_user [$time_local] ' '"$request" $status $body_bytes_sent "$http_referer" ' '"$http_user_agent" rt=$request_time $request_length $bytes_sent'; @@ -94,12 +107,12 @@ group => mirrors, require => User['mirrors']; - '/opt/mirrors/ftp/README.html': + '/opt/mirrors/ftp/test/README.html': source => 'puppet:///modules/ocf_mirrors/README.html', owner => mirrors, group => mirrors; - '/opt/mirrors/ftp/FOOTER.html': + '/opt/mirrors/ftp/test/FOOTER.html': source => 'puppet:///modules/ocf_mirrors/FOOTER.html', owner => mirrors, group => mirrors; @@ -129,20 +142,16 @@ raw_append => @(END), fancyindex on; fancyindex_name_length 100; + fancyindex_header /theme/header.html; + fancyindex_footer /theme/footer.html; fancyindex_exact_size off; - fancyindex_footer /FOOTER.html; - if ($http_user_agent ~ "(MSIE 7\.0; Windows NT (6\.1|6\.2)|Chrome\/34\.0|Chrome\/49\.0|Chrome\/67\.0|Edg\/85\.0\.537\.0)") { - return 403; - } + fancyindex_show_path off; END } nginx::resource::location { '= /': - ensure => present, - server => 'mirrors.ocf.berkeley.edu', - ssl => true, - raw_append => @(END), - fancyindex_header /README.html; - END + ensure => present, + server => 'mirrors.ocf.berkeley.edu', + ssl => true, } nginx::resource::location { '~ ^/tails': server => 'mirrors.ocf.berkeley.edu', @@ -152,12 +161,14 @@ etag off; END } - nginx::resource::location { '~ /\.(?!well-known).*': - ensure => present, - server => 'mirrors.ocf.berkeley.edu', - ssl => true, - raw_append => @(END), - deny all; + nginx::resource::location { '~ \.iso$': + server => 'mirrors.ocf.berkeley.edu', + ssl => true, + index_files => undef, + raw_append => @(END), + if ($http_user_agent = "curl/7.29.0") { + return 403; + } END } nginx::resource::server { 'mirrors.berkeley.edu': @@ -165,6 +176,7 @@ ipv6_enable => true, ipv6_listen_port => 80, www_root => '/var/www', + autoindex => 'on', location_cfg_append => { 'rewrite' => '^ http://mirrors.ocf.berkeley.edu permanent' } diff --git a/modules/ocf_mirrors/manifests/projects/almalinux.pp b/modules/ocf_mirrors/manifests/projects/almalinux.pp new file mode 100644 index 000000000..5b671f7d2 --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/almalinux.pp @@ -0,0 +1,17 @@ +class ocf_mirrors::projects::almalinux { + file { '/opt/mirrors/project/almalinux': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/almalinux/', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true, + } + + ocf_mirrors::timer { + 'almalinux': + exec_start => '/opt/mirrors/project/almalinux/sync-archive', + hour => '2/4', + minute => '43'; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/blackarch.pp b/modules/ocf_mirrors/manifests/projects/blackarch.pp new file mode 100644 index 000000000..5d9504fb9 --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/blackarch.pp @@ -0,0 +1,25 @@ +class ocf_mirrors::projects::blackarch { + file { + '/opt/mirrors/project/blackarch': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/blackarch/', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + + ocf_mirrors::monitoring { 'blackarch': + type => 'unix_timestamp', + upstream_host => 'blackarch.org', + upstream_path => '', + ts_path => 'lastupdate', + } + + ocf_mirrors::timer { + 'blackarch': + exec_start => '/opt/mirrors/project/blackarch/sync-archive', + hour => '0/2', + minute => '12'; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/blender.pp b/modules/ocf_mirrors/manifests/projects/blender.pp index cc254f555..a17ad0ab4 100644 --- a/modules/ocf_mirrors/manifests/projects/blender.pp +++ b/modules/ocf_mirrors/manifests/projects/blender.pp @@ -8,7 +8,14 @@ mode => '0755', recurse => true; } - + file { + '/opt/mirrors/project/blender/sync_password': + content => lookup('mirrors::blender_sync_password'), + show_diff => false, + owner => mirrors, + group => mirrors, + mode => '0400'; + } ocf_mirrors::timer { 'blender': exec_start => '/opt/mirrors/project/blender/sync-archive', diff --git a/modules/ocf_mirrors/manifests/projects/cran.pp b/modules/ocf_mirrors/manifests/projects/cran.pp new file mode 100644 index 000000000..1a7f50cad --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/cran.pp @@ -0,0 +1,17 @@ +class ocf_mirrors::projects::cran { + file { '/opt/mirrors/project/cran': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/cran/', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true, + } + + ocf_mirrors::timer { + 'cran': + exec_start => '/opt/mirrors/project/cran/sync-archive', + hour => '*', + minute => '20'; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/fedora.pp b/modules/ocf_mirrors/manifests/projects/fedora.pp index b98464bb5..276d4e75a 100644 --- a/modules/ocf_mirrors/manifests/projects/fedora.pp +++ b/modules/ocf_mirrors/manifests/projects/fedora.pp @@ -2,13 +2,30 @@ ocf_mirrors::qfm { 'epel': remote_host => 'rsync://dl.fedoraproject.org', - cron_hour => '0/6', + cron_hour => '0/4', cron_minute => '10'; } + + ocf_mirrors::monitoring { 'epel': + type => 'http_last_modified', + upstream_host => 'dl.fedoraproject.org', + ts_path => 'fullfiletimelist-epel', + upstream_path => '/pub/epel', + local_path => '/fedora/epel'; + } + ocf_mirrors::qfm { 'enchilada': - remote_host => 'rsync://mirrors.kernel.org', - cron_hour => '2/6', + remote_host => 'rsync://dl.fedoraproject.org', + cron_hour => '2/4', cron_minute => '20'; } + + ocf_mirrors::monitoring { 'enchilada': + type => 'http_last_modified', + upstream_host => 'dl.fedoraproject.org', + ts_path => 'fullfiletimelist-fedora', + upstream_path => '/pub/fedora', + local_path => '/fedora/fedora'; + } } diff --git a/modules/ocf_mirrors/manifests/projects/gimp.pp b/modules/ocf_mirrors/manifests/projects/gimp.pp new file mode 100644 index 000000000..dd12f090a --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/gimp.pp @@ -0,0 +1,28 @@ +class ocf_mirrors::projects::gimp { + file { + '/opt/mirrors/project/gimp': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/gimp', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + + file { + '/opt/mirrors/project/gimp/sync_password': + content => lookup('mirrors::gimp_sync_password'), + show_diff => false, + owner => mirrors, + group => mirrors, + mode => '0400'; + } + + ocf_mirrors::timer { + 'gimp': + exec_start => '/opt/mirrors/project/gimp/sync-archive', + hour => '0/6', + minute => '40', + require => File['/opt/mirrors/project/gimp']; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/ipfire.pp b/modules/ocf_mirrors/manifests/projects/ipfire.pp new file mode 100644 index 000000000..73e9f71ad --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/ipfire.pp @@ -0,0 +1,19 @@ +class ocf_mirrors::projects::ipfire { + file { + '/opt/mirrors/project/ipfire': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/ipfire', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + + ocf_mirrors::timer { + 'ipfire': + exec_start => '/opt/mirrors/project/ipfire/sync-archive', + hour => '1/6', + minute => '35', + require => File['/opt/mirrors/project/ipfire']; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/libreelec.pp b/modules/ocf_mirrors/manifests/projects/libreelec.pp new file mode 100644 index 000000000..6828a5a6f --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/libreelec.pp @@ -0,0 +1,18 @@ +class ocf_mirrors::projects::libreelec { + file { + '/opt/mirrors/project/libreelec': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/libreelec/', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + + ocf_mirrors::timer { + 'libreelec': + exec_start => '/opt/mirrors/project/libreelec/sync-archive', + hour => '0/6', + minute => '17'; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/linux_mint.pp b/modules/ocf_mirrors/manifests/projects/linux_mint.pp new file mode 100644 index 000000000..27c00901f --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/linux_mint.pp @@ -0,0 +1,19 @@ +class ocf_mirrors::projects::linux_mint { + file { + '/opt/mirrors/project/linux-mint': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/linux-mint', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + + ocf_mirrors::timer { + 'linux-mint': + exec_start => '/opt/mirrors/project/linux-mint/sync-archive', + hour => '0/12', + minute => '25', + require => File['/opt/mirrors/project/linux-mint']; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/linuxmint_packages.pp b/modules/ocf_mirrors/manifests/projects/linuxmint_packages.pp new file mode 100644 index 000000000..7ad4ae53e --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/linuxmint_packages.pp @@ -0,0 +1,27 @@ +class ocf_mirrors::projects::linuxmint_packages { + file { + '/opt/mirrors/project/linuxmint-packages': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/linuxmint-packages', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + + ocf_mirrors::monitoring { + 'linuxmint-packages': + type => 'debian', + dist_to_check => 'una', + upstream_host => 'packages.linuxmint.com', + upstream_path => '/'; + } + + ocf_mirrors::timer { + 'linuxmint-packages': + exec_start => '/opt/mirrors/project/linuxmint-packages/sync-archive', + hour => '0/6', + minute => '40', + require => File['/opt/mirrors/project/linuxmint-packages']; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/mx_linux.pp b/modules/ocf_mirrors/manifests/projects/mx_linux.pp new file mode 100644 index 000000000..0a7b59ac5 --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/mx_linux.pp @@ -0,0 +1,26 @@ +class ocf_mirrors::projects::mx_linux { + file { + '/opt/mirrors/project/mx-linux': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/mx-linux', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + file { + '/opt/mirrors/project/mx-linux/sync_password': + content => lookup('mirrors::mx_linux_sync_password'), + show_diff => false, + owner => mirrors, + group => mirrors, + mode => '0400'; + } + ocf_mirrors::timer { + 'mx-linux': + exec_start => '/opt/mirrors/project/mx-linux/sync-archive', + hour => '1/6', + minute => '35', + require => File['/opt/mirrors/project/mx-linux']; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/mx_packages.pp b/modules/ocf_mirrors/manifests/projects/mx_packages.pp new file mode 100644 index 000000000..98bbe10aa --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/mx_packages.pp @@ -0,0 +1,26 @@ +class ocf_mirrors::projects::mx_packages { + file { + '/opt/mirrors/project/mx-packages': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/mx-packages', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + file { + '/opt/mirrors/project/mx-packages/sync_password': + content => lookup('mirrors::mx_packages_sync_password'), + show_diff => false, + owner => mirrors, + group => mirrors, + mode => '0400'; + } + ocf_mirrors::timer { + 'mx-packages': + exec_start => '/opt/mirrors/project/mx-packages/sync-archive', + hour => '1/2', + minute => '35', + require => File['/opt/mirrors/project/mx-packages']; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/openwrt.pp b/modules/ocf_mirrors/manifests/projects/openwrt.pp new file mode 100644 index 000000000..dc6b9e7de --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/openwrt.pp @@ -0,0 +1,19 @@ +class ocf_mirrors::projects::openwrt { + file { + '/opt/mirrors/project/openwrt': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/openwrt', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + + ocf_mirrors::timer { + 'openwrt': + exec_start => '/opt/mirrors/project/openwrt/sync-archive', + hour => '3/6', + minute => '21', + require => File['/opt/mirrors/project/openwrt']; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/osdn.pp b/modules/ocf_mirrors/manifests/projects/osdn.pp new file mode 100644 index 000000000..bd518ba60 --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/osdn.pp @@ -0,0 +1,19 @@ +class ocf_mirrors::projects::osdn { + file { + '/opt/mirrors/project/osdn': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/osdn', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + + ocf_mirrors::timer { + 'osdn': + exec_start => '/opt/mirrors/project/osdn/sync-archive', + hour => '*', + minute => '32', + require => File['/opt/mirrors/project/osdn']; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/pikvm.pp b/modules/ocf_mirrors/manifests/projects/pikvm.pp new file mode 100644 index 000000000..a8599ca2d --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/pikvm.pp @@ -0,0 +1,17 @@ +class ocf_mirrors::projects::pikvm { + file { + '/opt/mirrors/project/pikvm': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/pikvm/', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + ocf_mirrors::timer { + 'pikvm': + exec_start => '/opt/mirrors/project/pikvm/sync-archive', + hour => '0/12', + minute => '30'; + } +} diff --git a/modules/ocf_mirrors/manifests/projects/raspbian.pp b/modules/ocf_mirrors/manifests/projects/raspbian.pp index 2cb740694..447892bb3 100644 --- a/modules/ocf_mirrors/manifests/projects/raspbian.pp +++ b/modules/ocf_mirrors/manifests/projects/raspbian.pp @@ -7,7 +7,7 @@ ocf_mirrors::monitoring { 'raspbian': type => 'debian', - dist_to_check => 'stretch', + dist_to_check => 'bullseye', local_path => '/raspbian/raspbian', upstream_host => 'archive.raspbian.org'; } diff --git a/modules/ocf_mirrors/manifests/projects/raspi.pp b/modules/ocf_mirrors/manifests/projects/raspi.pp index 9f6db92bb..44a05b61f 100644 --- a/modules/ocf_mirrors/manifests/projects/raspi.pp +++ b/modules/ocf_mirrors/manifests/projects/raspi.pp @@ -9,6 +9,14 @@ recurse => true; } + ocf_mirrors::monitoring { 'raspi': + type => 'debian', + dist_to_check => 'bullseye', + local_path => '/raspi/debian', + upstream_host => 'archive.raspberrypi.org', + upstream_path => '/debian'; + } + ocf_mirrors::timer { 'raspi': exec_start => '/opt/mirrors/project/raspi/sync-archive', diff --git a/modules/ocf_mirrors/manifests/projects/rocky.pp b/modules/ocf_mirrors/manifests/projects/rocky.pp index 5478d7ff7..723fcd6ed 100644 --- a/modules/ocf_mirrors/manifests/projects/rocky.pp +++ b/modules/ocf_mirrors/manifests/projects/rocky.pp @@ -8,6 +8,13 @@ recurse => true, } + ocf_mirrors::monitoring { 'rocky': + type => 'http_last_modified', + upstream_host => 'download.rockylinux.org', + ts_path => 'fullfiletimelist-rocky', + upstream_path => '/pub/rocky', + } + ocf_mirrors::timer { 'rocky': exec_start => '/opt/mirrors/project/rocky/sync-archive', diff --git a/modules/ocf_mirrors/manifests/projects/rpmfusion.pp b/modules/ocf_mirrors/manifests/projects/rpmfusion.pp new file mode 100644 index 000000000..64fcd0147 --- /dev/null +++ b/modules/ocf_mirrors/manifests/projects/rpmfusion.pp @@ -0,0 +1,19 @@ +class ocf_mirrors::projects::rpmfusion { + file { + '/opt/mirrors/project/rpmfusion': + ensure => directory, + source => 'puppet:///modules/ocf_mirrors/project/rpmfusion', + owner => mirrors, + group => mirrors, + mode => '0755', + recurse => true; + } + + ocf_mirrors::timer { + 'rpmfusion': + exec_start => '/opt/mirrors/project/rpmfusion/sync-archive', + hour => '2/6', + minute => '15', + require => File['/opt/mirrors/project/rpmfusion']; + } +}