Merge branch '3.0-dev' into 3.0

microsoft · Jan 18, 2025 · 417be0b · 417be0b
2 parents 2a17e48 + 6e62dc3
commit 417be0b
Show file tree

Hide file tree

Showing 189 changed files with 11,271 additions and 1,230 deletions.
diff --git a/.CodeQL.yml b/.CodeQL.yml
@@ -0,0 +1,29 @@
+path_classifiers:
+  library:
+    # Treat source files for all compiled languages in the specs directories
+    # as 3rd party library sources because they are not owned by us.
+    #
+    # Extensions from https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/.
+    - "SPECS*/**/*.c"
+    - "SPECS*/**/*.c++"
+    - "SPECS*/**/*.cc"
+    - "SPECS*/**/*.cpp"
+    - "SPECS*/**/*.cs"
+    - "SPECS*/**/*.cshtml"
+    - "SPECS*/**/*.csproj"
+    - "SPECS*/**/*.cts"
+    - "SPECS*/**/*.cxx"
+    - "SPECS*/**/*.go"
+    - "SPECS*/**/*.h"
+    - "SPECS*/**/*.h++"
+    - "SPECS*/**/*.hh"
+    - "SPECS*/**/*.hpp"
+    - "SPECS*/**/*.hxx"
+    - "SPECS*/**/*.java"
+    - "SPECS*/**/*.kt"
+    - "SPECS*/**/*.mts"
+    - "SPECS*/**/*.sln"
+    - "SPECS*/**/*.swift"
+    - "SPECS*/**/*.ts"
+    - "SPECS*/**/*.tsx"
+    - "SPECS*/**/*.xaml"
diff --git a/.github/workflows/check-manifests.yml b/.github/workflows/check-manifests.yml
@@ -21,7 +21,9 @@ jobs:
     # This PR runner uses an older Ubuntu with rpm version 4.17, which doesn't understand some newer macros like %bcond
     - name: Define missing rpm macros
       run: |
-        [[ -n $(rpm --eval '%bcond test 1') ]] && echo '%bcond() %[ (%{2}) ? "%{expand:%%bcond_without %{1}}" : "%{expand:%%bcond_with %{1}}" ]' > ~/.rpmmacros
+        if [[ -n $(rpm --eval '%bcond test 1') ]]; then
+          echo '%bcond() %[ (%{2}) ? "%{expand:%%bcond_without %{1}}" : "%{expand:%%bcond_with %{1}}" ]' > ~/.rpmmacros
+        fi
 
     - name: Check x86_64 manifests
       run: |

diff --git a/.github/workflows/check-package-cgmanifest.yml b/.github/workflows/check-package-cgmanifest.yml
@@ -22,7 +22,9 @@ jobs:
     # This PR runner uses an older Ubuntu with rpm version 4.17, which doesn't understand some newer macros like %bcond
     - name: Define missing rpm macros
       run: |
-        [[ -n $(rpm --eval '%bcond test 1') ]] && echo '%bcond() %[ (%{2}) ? "%{expand:%%bcond_without %{1}}" : "%{expand:%%bcond_with %{1}}" ]' > ~/.rpmmacros
+        if [[ -n $(rpm --eval '%bcond test 1') ]]; then
+          echo '%bcond() %[ (%{2}) ? "%{expand:%%bcond_without %{1}}" : "%{expand:%%bcond_with %{1}}" ]' > ~/.rpmmacros
+        fi
 
     - name: Get base commit for PRs
       if: ${{ github.event_name == 'pull_request' }}

diff --git a/.pipelines/containerSourceData/valkey/Dockerfile-Valkey b/.pipelines/containerSourceData/valkey/Dockerfile-Valkey
@@ -0,0 +1,26 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+ARG BASE_IMAGE
+
+FROM $BASE_IMAGE
+
+@INCLUDE_MAIN_RUN_INSTRUCTION@
+
+RUN set -eux && \
+    valkey-cli --version && \
+    valkey-server --version && \
+    mkdir /data && \
+    chown valkey:valkey /data
+
+VOLUME /data
+WORKDIR /data
+
+COPY valkey-docker-entrypoint.sh /usr/local/bin/
+
+RUN chmod +x /usr/local/bin/valkey-docker-entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/valkey-docker-entrypoint.sh"]
+
+EXPOSE 6379
+CMD ["valkey-server"]
diff --git a/.pipelines/containerSourceData/valkey/configuration-files/valkey-docker-entrypoint.sh b/.pipelines/containerSourceData/valkey/configuration-files/valkey-docker-entrypoint.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+set -e
+
+# first arg is `-f` or `--some-option`
+# or first arg is `something.conf`
+if [ "${1#-}" != "$1" ] || [ "${1%.conf}" != "$1" ]; then
+    set -- valkey-server "$@"
+fi
+
+# allow the container to be started with `--user`
+if [ "$1" = 'valkey-server' -a "$(id -u)" = '0' ]; then
+    find . \! -user valkey -exec chown valkey '{}' +
+    exec setpriv --reuid=valkey --regid=valkey --init-groups --inh-caps=-all "$BASH_SOURCE" "$@"
+fi
+
+# set an appropriate umask (if one isn't set already)
+# - https://github.com/docker-library/redis/issues/305
+# - https://github.com/redis/redis/blob/bb875603fb7ff3f9d19aad906bd45d7db98d9a39/utils/systemd-redis_server.service#L37
+um="$(umask)"
+if [ "$um" = '0022' ]; then
+	umask 0077
+fi
+
+exec "$@" $VALKEY_EXTRA_FLAGS
diff --git a/.pipelines/containerSourceData/valkey/valkey.name b/.pipelines/containerSourceData/valkey/valkey.name
@@ -0,0 +1 @@
+valkey
diff --git a/.pipelines/containerSourceData/valkey/valkey.pkg b/.pipelines/containerSourceData/valkey/valkey.pkg
@@ -0,0 +1,3 @@
+valkey
+cronie
+util-linux
diff --git a/.pipelines/prchecks/PackageBuildPRCheck.yml b/.pipelines/prchecks/PackageBuildPRCheck.yml
@@ -80,9 +80,11 @@ extends:
                       # GCC fails to build as a regular package.
                       ignoredSpecs: ["gcc"]
 
-                  - script: echo "##vso[task.setvariable variable=toolchainArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                  - script: |
+                      echo "##vso[task.setvariable variable=toolchainArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                      echo "##vso[task.setvariable variable=toolchainTarballName;isOutput=true]toolchain_built_rpms_all.tar.gz"
                     name: "ToolchainArtifactName"
-                    displayName: "Set variable for published artifact name"
+                    displayName: "Set variables for published toolchain tarball"
 
                   # 1. Automatic publishing won't work if 'isCustom: true' is set on the pool. We cannot do 'isCustom: false' because
                   #    then OneBranch attempts to perform additional actions (adding build tags for instance), which require additional permissions
@@ -104,24 +106,38 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
+                  inputArtifactsLocation: $(Agent.TempDirectory)
                   ob_artifactBaseName: $(rpmsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                   ob_outputDirectory: $(Build.ArtifactStagingDirectory)
+                  outputRPMsTarballName: "rpms.tar.gz"
                   toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
+                  toolchainTarballName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainTarballName'] ]
                 steps:
+                  - task: DownloadPipelineArtifact@2
+                    displayName: "Download toolchain"
+                    inputs:
+                      artifact: $(toolchainArtifactName)
+                      patterns: "**/$(toolchainTarballName)"
+                      targetPath: $(inputArtifactsLocation)
+
                   - template: .pipelines/templates/PackageBuild.yml@self
                     parameters:
                       checkBuildRetries: "1"
-                      customToolchainArtifactName: $(toolchainArtifactName)
+                      customToolchainTarballName: $(toolchainTarballName)
+                      inputArtifactsFolder: $(inputArtifactsLocation)
                       isCheckBuild: true
                       isQuickRebuildPackages: true
                       isUseCCache: true
                       maxCPU: "${{ configuration.maxCPUs }}"
                       outputArtifactsFolder: $(ob_outputDirectory)
+                      outputRPMsTarballName: $(outputRPMsTarballName)
                       pipArtifactFeeds: "mariner/Mariner-Pypi-Feed"
                       selfRepoName: self
                       testSuiteName: "[${{ configuration.name }}] Package test"
 
-                  - script: echo "##vso[task.setvariable variable=rpmsArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                  - script: |
+                      echo "##vso[task.setvariable variable=rpmsArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                      echo "##vso[task.setvariable variable=rpmsTarballName;isOutput=true]$(outputRPMsTarballName)"
                     name: "RPMsArtifactName"
                     displayName: "Set variable for published artifact name"
 
@@ -142,15 +158,25 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
+                  inputArtifactsLocation: $(Agent.TempDirectory)
                   ob_artifactBaseName: $(toolchainTestsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                   ob_outputDirectory: $(Build.ArtifactStagingDirectory)
                   testListFromToolchain: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList'] ]
                   toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
+                  toolchainTarballName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainTarballName'] ]
                 steps:
+                  - task: DownloadPipelineArtifact@2
+                    displayName: "Download toolchain"
+                    inputs:
+                      artifact: $(toolchainArtifactName)
+                      patterns: "**/$(toolchainTarballName)"
+                      targetPath: $(inputArtifactsLocation)
+
                   - template: .pipelines/templates/PackageBuild.yml@self
                     parameters:
                       checkBuildRetries: "1"
-                      customToolchainArtifactName: $(toolchainArtifactName)
+                      customToolchainTarballName: $(toolchainTarballName)
+                      inputArtifactsFolder: $(inputArtifactsLocation)
                       isAllowToolchainRebuilds: true
                       isCheckBuild: true
                       isQuickRebuildPackages: true
@@ -179,8 +205,18 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
+                  inputArtifactsLocation: $(Agent.TempDirectory)
                   rpmsArtifactName: $[ stageDependencies.RPMs_${{ configuration.name }}.BuildAndTest.outputs['RPMsArtifactName.rpmsArtifactName'] ]
+                  rpmsTarballName: $[ stageDependencies.RPMs_${{ configuration.name }}.BuildAndTest.outputs['RPMsArtifactName.rpmsTarballName'] ]
                 steps:
+                  - task: DownloadPipelineArtifact@2
+                    displayName: "Download RPMs tarball"
+                    inputs:
+                      artifact: $(rpmsArtifactName)
+                      patterns: "**/$(rpmsTarballName)"
+                      targetPath: $(inputArtifactsLocation)
+
                   - template: .pipelines/templatesWithCheckout/SodiffCheck.yml@self
                     parameters:
-                      inputArtifactName: $(rpmsArtifactName)
+                      inputArtifactsFolder: $(inputArtifactsLocation)
+                      inputRPMsTarballName: $(rpmsTarballName)
diff --git a/.pipelines/templates/PackageBuild.yml b/.pipelines/templates/PackageBuild.yml
@@ -14,13 +14,9 @@ parameters:
     type: number
     default: 12
 
-  - name: customToolchainArtifactName
-    type: string
-    default: ""
-
   - name: customToolchainTarballName
     type: string
-    default: "toolchain_built_rpms_all.tar.gz"
+    default: ""
 
   - name: extraPackageRepos
     type: string
@@ -30,12 +26,16 @@ parameters:
     type: boolean
     default: true
 
-  - name: inputCacheArtifacts
+  - name: inputArtifactsFolder
+    type: string
+    default: "$(Agent.TempDirectory)"
+
+  - name: inputCacheRPMsTarballs
     type: object
     default: []
     # Sample:
-    # - name: build-artifacts
-    #   rpmsTarball: cache.tar.gz
+    #   - cache.tar.gz
+    #   - cache2.tar.gz
 
   - name: isAllowToolchainRebuilds
     type: string
@@ -160,15 +160,9 @@ steps:
           artifactFeeds: "${{ parameters.pipArtifactFeeds }}"
         displayName: "Authenticate to custom pip artifact feeds"
 
-  - ${{ if parameters.customToolchainArtifactName }}:
-      - task: DownloadPipelineArtifact@2
-        displayName: "Download toolchain"
-        inputs:
-          artifact: "${{ parameters.customToolchainArtifactName }}"
-          patterns: "**/${{ parameters.customToolchainTarballName }}"
-
+  - ${{ if parameters.customToolchainTarballName }}:
       - script: |
-          toolchain_archive="$(find "$(Pipeline.Workspace)" -name "${{ parameters.customToolchainTarballName }}" -print -quit)"
+          toolchain_archive="$(find "${{ parameters.inputArtifactsFolder }}" -name "${{ parameters.customToolchainTarballName }}" -print -quit)"
           if [[ ! -f "$toolchain_archive" ]]; then
             echo "ERROR: toolchain archive not found!" >&2
             exit 1
@@ -178,17 +172,11 @@ steps:
           sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" toolchain TOOLCHAIN_ARCHIVE="$toolchain_archive"
         displayName: "Populate toolchain"
 
-  - ${{ each inputCacheArtifact in parameters.inputCacheArtifacts }}:
-      - task: DownloadPipelineArtifact@2
-        displayName: "Download input cache RPM from ${{ inputCacheArtifact.name }}"
-        inputs:
-          artifact: "${{ inputCacheArtifact.name }}"
-          patterns: "**/${{ inputCacheArtifact.rpmsTarball }}"
-
+  - ${{ each inputCacheRPMsTarball in parameters.inputCacheRPMsTarballs }}:
       - script: |
-          rpms_archive="$(find "$(Pipeline.Workspace)" -name "${{ inputCacheArtifact.rpmsTarball }}" -print -quit)"
+          rpms_archive="$(find "${{ parameters.inputArtifactsFolder }}" -name "${{ inputCacheRPMsTarball }}" -print -quit)"
           if [[ ! -f "$rpms_archive" ]]; then
-            echo "ERROR: cache RPMs archive '${{ inputCacheArtifact.rpmsTarball }}' not found!" >&2
+            echo "ERROR: cache RPMs archive '${{ inputCacheRPMsTarball }}' not found!" >&2
             exit 1
           fi
 
@@ -200,7 +188,7 @@ steps:
         check_build_retries_arg="CHECK_BUILD_RETRIES=${{ parameters.checkBuildRetries }}"
       fi
 
-      if [[ -n "${{ parameters.customToolchainArtifactName }}" ]]; then
+      if [[ -n "${{ parameters.customToolchainTarballName }}" ]]; then
         toolchain_archive_arg="TOOLCHAIN_ARCHIVE=$(toolchainArchive)"
       fi
 

diff --git a/.pipelines/templatesWithCheckout/SodiffCheck.yml b/.pipelines/templatesWithCheckout/SodiffCheck.yml
@@ -6,8 +6,9 @@ parameters:
     type: string
     default: "$(Build.SourcesDirectory)"
 
-  - name: inputArtifactName
+  - name: inputArtifactsFolder
     type: string
+    default: "$(Agent.TempDirectory)"
 
   - name: inputRPMsTarballName
     type: string
@@ -26,19 +27,11 @@ parameters:
     default: "$(Agent.TempDirectory)/SourcesWorkspace"
 
 steps:
-  - task: DownloadPipelineArtifact@2
-    displayName: "Download sources for signing"
-    inputs:
-      artifact: ${{ parameters.inputArtifactName }}
-      patterns: |
-        **/${{ parameters.inputRPMsTarballName }}
-      targetPath: "$(Agent.TempDirectory)"
-
   - script: |
       set -e
 
       mkdir -p "${{ parameters.sourcesWorkspace }}"
-      find "$(Agent.TempDirectory)" -name "${{ parameters.inputRPMsTarballName }}" -print0 | xargs -0 -n 1 tar -C "${{ parameters.sourcesWorkspace }}" -xkf
+      find "${{ parameters.inputArtifactsFolder }}" -name "${{ parameters.inputRPMsTarballName }}" -print0 | xargs -0 -n 1 tar -C "${{ parameters.sourcesWorkspace }}" -xkf
     displayName: "Extract sources tarball"
 
   - script: |

diff --git a/LICENSES-AND-NOTICES/SPECS/LICENSES-MAP.md b/LICENSES-AND-NOTICES/SPECS/LICENSES-MAP.md
diff --git a/LICENSES-AND-NOTICES/SPECS/data/licenses.json b/LICENSES-AND-NOTICES/SPECS/data/licenses.json
@@ -548,6 +548,7 @@
                 "kata-containers",
                 "kde-filesystem",
                 "kde-settings",
+                "kernel-srpm-macros",
                 "kexec-tools",
                 "keybinder3",
                 "keycloak-httpd-client-install",
@@ -1764,6 +1765,7 @@
                 "python-poetry-core",
                 "python-process-tests",
                 "python-productmd",
+                "python-prometheus_client",
                 "python-ptyprocess",
                 "python-pycares",
                 "python-pycosat",
@@ -2454,12 +2456,46 @@
         "NVIDIA": {
             "license": "[ASL 2.0 License and spec specific licenses](http://www.apache.org/licenses/LICENSE-2.0)",
             "specs": [
+                "fwctl",
+                "fwctl-signed",
+                "ibarr",
+                "ibsim",
+                "iser",
+                "iser-signed",
+                "isert",
+                "isert-signed",
+                "knem",
+                "knem-modules-signed",
                 "libnvidia-container",
+                "mft_kernel",
+                "mft_kernel-signed",
+                "mlnx-ethtool",
+                "mlnx-iproute2",
+                "mlnx-nfsrdma",
+                "mlnx-nfsrdma-signed",
+                "mlnx-ofa_kernel",
+                "mlnx-ofa_kernel-modules-signed",
                 "mlnx-tools",
                 "mlx-bootctl",
+                "mlx-steering-dump",
+                "multiperf",
                 "nvidia-container-toolkit",
+                "ofed-docs",
                 "ofed-scripts",
-                "perftest"
+                "perftest",
+                "rshim",
+                "sockperf",
+                "srp",
+                "srp-signed",
+                "xpmem",
+                "xpmem-lib",
+                "xpmem-modules-signed"
+            ]
+        },
+        "NVIDIA (BSD)": {
+            "license": "[BSD](https://github.com/Mellanox/sockperf/blob/sockperf_v2/copying)",
+            "specs": [
+                "sockperf"
             ]
         },
         "OpenEuler": {