tests/nested/manual/hybrid-remodel: use fakestore

tests/lib/assertions/developer1-22-classic-dangerous-rev1.json

@@ -0,0 +1,49 @@
+{
+    "type": "model",
+    "authority-id": "developer1",
+    "series": "16",
+    "brand-id": "developer1",
+    "model": "developer1-22-classic-dangerous",
+    "architecture": "amd64",
+    "timestamp": "2022-09-06T22:00:00+00:00",
+    "grade": "dangerous",
+    "base": "core22",
+    "classic": "true",
+    "distribution": "ubuntu",
+    "revision": "1",
+    "serial-authority": [
+        "generic"
+    ],
+    "snaps": [
+        {
+            "default-channel": "22/edge",
+            "id": "UqFziVZDHLSyO3TqSWgNBoAdHbLI4dAH",
+            "name": "pc",
+            "type": "gadget"
+        },
+        {
+            "default-channel": "22-oem/stable",
+            "id": "pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza",
+            "name": "pc-kernel",
+            "type": "kernel"
+        },
+        {
+            "default-channel": "latest/edge",
+            "id": "amcUKQILKXHHTlmSa7NMdnXSx02dNeeT",
+            "name": "core22",
+            "type": "base"
+        },
+        {
+            "default-channel": "latest/stable",
+            "id": "PMrrV4ml8uWuEUDBT8dSGnKUYbevVhc4",
+            "name": "snapd",
+            "type": "snapd"
+        },
+        {
+            "default-channel": "latest/edge",
+            "id": "vFTrGWyktbEDwuYJjdsZykV270mqcNz8",
+            "name": "jq-core22",
+            "type": "app"
+        }
+    ]
+}

tests/lib/assertions/developer1-24-classic-dangerous-rev1.json

@@ -0,0 +1,49 @@
+{
+    "type": "model",
+    "authority-id": "developer1",
+    "series": "16",
+    "brand-id": "developer1",
+    "model": "developer1-24-classic-dangerous",
+    "architecture": "amd64",
+    "timestamp": "2024-04-09T22:00:00+00:00",
+    "grade": "dangerous",
+    "base": "core24",
+    "classic": "true",
+    "distribution": "ubuntu",
+    "revision": "1",
+    "serial-authority": [
+        "generic"
+    ],
+    "snaps": [
+        {
+            "default-channel": "24/edge",
+            "id": "UqFziVZDHLSyO3TqSWgNBoAdHbLI4dAH",
+            "name": "pc",
+            "type": "gadget"
+        },
+        {
+            "default-channel": "24-oem/stable",
+            "id": "pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza",
+            "name": "pc-kernel",
+            "type": "kernel"
+        },
+        {
+            "default-channel": "latest/edge",
+            "id": "dwTAh7MZZ01zyriOZErqd1JynQLiOGvM",
+            "name": "core24",
+            "type": "base"
+        },
+        {
+            "default-channel": "latest/stable",
+            "id": "PMrrV4ml8uWuEUDBT8dSGnKUYbevVhc4",
+            "name": "snapd",
+            "type": "snapd"
+        },
+        {
+            "default-channel": "latest/stable",
+            "id": "zBAzEImJzOAoGxmoG5o6Tl6ELZXMeUlt",
+            "name": "test-snapd-jq-core24",
+            "type": "app"
+        }
+    ]
+}

tests/lib/tools/setup_nested_hybrid_system.sh

@@ -64,7 +64,9 @@ run_muinstaller() {
     fi
 
     # build the muinstaller snap
-    snap install snapcraft --candidate --classic
+    if [ -z "$(command -v snapcraft)" ]; then
+        snap install snapcraft --candidate --classic
+    fi
     "${TESTSTOOLS}/lxd-state" prepare-snap
     (cd "${TESTSLIB}/muinstaller" && snapcraft)
 
@@ -99,6 +101,13 @@ run_muinstaller() {
     done
 
     remote.exec "sudo sh -c 'echo SNAPD_DEBUG=1 >> /etc/environment'"
+    if [ -n "${HYBRID_SYSTEM_SNAPD_ENVIRONMENT-}" ]; then
+        remote.exec "sudo mkdir -p /etc/systemd/system/snapd.service.d"
+        cat <<EOF | remote.exec "sudo tee /etc/systemd/system/snapd.service.d/hybrid-system-environment.conf"
+[Service]
+Environment=${HYBRID_SYSTEM_SNAPD_ENVIRONMENT-}
+EOF
+    fi
     # push our snap down
     # TODO: this abuses /var/lib/snapd to store the deb so that mk-initramfs-classic
     # can pick it up. the real installer will also need a very recent snapd
@@ -147,10 +156,25 @@ run_muinstaller() {
     # run installation
     local install_disk
     install_disk=$(remote.exec "readlink -f /dev/disk/by-id/virtio-target")
+
+    if [ -n "${HYBRID_SYSTEM_MK_ROOT_FS-}" ]; then
+        remote.push "${HYBRID_SYSTEM_MK_ROOT_FS}" /home/user1/custom-rootfs.sh
+        remote.exec "chmod +x /home/user1/custom-rootfs.sh"
+    fi
+    remote.exec "tee /home/user1/mk-classic-rootfs-wrapper.sh" <<\EOF
+#!/bin/bash
+set -eu
+/snap/muinstaller/current/bin/mk-classic-rootfs.sh "$@"
+if [ -x /home/user1/custom-rootfs.sh ]; then
+  /home/user1/custom-rootfs.sh "$@"
+fi
+EOF
+    remote.exec "chmod +x /home/user1/mk-classic-rootfs-wrapper.sh"
+
     remote.exec "sudo muinstaller \
         -label ${label} \
         -device ${install_disk} \
-        -rootfs-creator /snap/muinstaller/current/bin/mk-classic-rootfs.sh"
+        -rootfs-creator /home/user1/mk-classic-rootfs-wrapper.sh"
 
     remote.exec "sudo sync"
 

tests/nested/manual/hybrid-remodel/prepare-device

@@ -0,0 +1,3 @@
+#!/bin/sh
+# 10.0.2.2 is the host from a nested VM
+snapctl set device-service.url=http://10.0.2.2:11029

tests/nested/manual/hybrid-remodel/prepare-rootfs.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -eu
+
+rootfs="$1"
+
+mkdir -p "${rootfs}/etc/systemd/system/snapd.service.d"
+cat <<EOF >"${rootfs}/etc/systemd/system/snapd.service.d/snapd-override.conf"
+[Service]
+Environment=SNAPPY_FORCE_API_URL=http://10.0.2.2:11028
+Environment=SNAPD_DEBUG=1
+Environment=SNAPPY_USE_STAGING_STORE=0
+Environment=SNAPPY_TESTING=1
+EOF

tests/nested/manual/hybrid-remodel/task.yaml

@@ -11,45 +11,150 @@ environment:
   NESTED_REPACK_KERNEL_SNAP: true
   NESTED_ENABLE_OVMF: true
 
-  STORE_ADDR: localhost:11028
-  STORE_DIR: $(pwd)/fake-store-blobdir
+  NESTED_SIGN_SNAPS_FAKESTORE: true
+  # for the fake store
+  NESTED_FAKESTORE_BLOB_DIR: $(pwd)/fake-store-blobdir
+  NESTED_UBUNTU_IMAGE_SNAPPY_FORCE_SAS_URL: http://localhost:11028
 
 prepare: |
   VERSION="$(tests.nested show version)"
 
-  # download beta to increase the unlikeliness that we encounter the
-  # same revision when remodelling. 
-  snap download --basename=pc-kernel --channel="$VERSION/beta" pc-kernel
+  snap install remarshal lxd
+  snap install --edge test-snapd-swtpm
+  snap install snapcraft --classic
+
+  case "${VERSION}" in
+    22)
+      snap_app_name=jq-core22
+      snap_app_id="vFTrGWyktbEDwuYJjdsZykV270mqcNz8"
+      app_channel="latest/edge"
+    ;;
+    24)
+      snap_app_name=test-snapd-jq-core24
+      snap_app_id="zBAzEImJzOAoGxmoG5o6Tl6ELZXMeUlt"
+      app_channel="latest/stable"
+    ;;
+  esac
+  snap download --channel="${app_channel}" "${snap_app_name}" --basename="${snap_app_name}"
+
+  "${TESTSTOOLS}/store-state" setup-fake-store "${NESTED_FAKESTORE_BLOB_DIR}"
+  for key in "${TESTSLIB}"/assertions/{testrootorg-store.account-key,developer1.account,developer1.account-key}; do
+    cp "${key}" "${NESTED_FAKESTORE_BLOB_DIR}/asserts"
+    snap ack "${key}"
+  done
+
+  "${TESTSTOOLS}"/store-state make-snap-installable --noack "${NESTED_FAKESTORE_BLOB_DIR}" "${snap_app_name}.snap" "${snap_app_id}"
+
+  tests.nested prepare_essential_snaps
+
+  extra_snaps="$(tests.nested get extra-snaps-path)"
+
+  # shellcheck source=tests/lib/prepare.sh
+  . "$TESTSLIB/prepare.sh"
+
+  case "${VERSION}" in
+    22)
+      # use 23.10 instead
+      snap download pc-kernel --channel=23.10/beta  --basename="pc-kernel-23.10"
+      rm -rf pc-kernel-repacked/
+      mkdir -p pc-kernel-repacked/
+      uc20_build_initramfs_kernel_snap pc-kernel-23.10.snap pc-kernel-repacked/
+      mv pc-kernel-repacked/pc-kernel_*.snap "${extra_snaps}/pc-kernel-23.10.snap"
+      rmdir pc-kernel-repacked
+      pc_kernel_orig="${extra_snaps}/pc-kernel-23.10.snap"
+      "${TESTSTOOLS}"/store-state make-snap-installable --noack --revision 2 "${NESTED_FAKESTORE_BLOB_DIR}" "${pc_kernel_orig}" "pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza"
+    ;;
+    *)
+      pc_kernel_orig="${extra_snaps}/pc-kernel.snap"
+    ;;
+  esac
+
+  unsquashfs -d pc-kernel "${pc_kernel_orig}"
+
+  sed -i 's/^\(version: .*\)/\1-oem/' pc-kernel/meta/snap.yaml
+
+  # Re-sign the kernel to force a reseal
+  # shellcheck source=tests/lib/nested.sh
+  . "$TESTSLIB/nested.sh"
+  KEY_NAME=$(nested_get_snakeoil_key)
+  sha256sum pc-kernel/kernel.efi
+  SNAKEOIL_KEY="$PWD/$KEY_NAME.key"
+  SNAKEOIL_CERT="$PWD/$KEY_NAME.pem"
+  nested_secboot_sign_kernel pc-kernel "${SNAKEOIL_KEY}" "${SNAKEOIL_CERT}"
+  sha256sum pc-kernel/kernel.efi
+
+  snap pack pc-kernel --filename="${extra_snaps}/pc-kernel-oem.snap"
+
+  rm -rf pc-kernel
+  "${TESTSTOOLS}"/store-state make-snap-installable --noack --revision 3 "${NESTED_FAKESTORE_BLOB_DIR}" "${extra_snaps}/pc-kernel-oem.snap" "pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza"
+
   if [ "$VERSION" == 22 ]; then
     PC_VERSION="23.10"
   else
     PC_VERSION="${VERSION}.04"
   fi
   snap download --basename=pc --channel="classic-${PC_VERSION}/stable" pc
 
-  if [ "$VERSION" == 24 ]; then
-    # there was made a decision to use core22 as base for
-    # the pc gadget here, we do not want that when testing
-    unsquashfs -d pc-gadget pc.snap
-    rm pc.snap
-
-    sed -i 's/core22/core24/g' pc-gadget/snap/snapcraft.yaml
-    sed -i 's/core22/core24/g' pc-gadget/meta/snap.yaml
-    snap pack pc-gadget
-    rm -rf pc-gadget
-    mv pc_*.snap pc.snap
-  fi
-
+  unsquashfs -d pc pc.snap
+  rm -rf pc.snap
+  case "${VERSION}" in
+    24)
+      sed -i 's/^base: core22$/base: core24/' pc/meta/snap.yaml
+    ;;
+  esac
+  mkdir -p pc/meta/hooks/
+  cp prepare-device pc/meta/hooks/
+  chmod +x pc/meta/hooks/prepare-device
+  echo "console=ttyS0 systemd.journald.forward_to_console=1" >>pc/cmdline.extra
+  snap pack pc --filename="${extra_snaps}/pc-prepare-device.snap"
+  rm -rf pc
+  "${TESTSTOOLS}"/store-state make-snap-installable --noack --revision 2 "${NESTED_FAKESTORE_BLOB_DIR}" "${extra_snaps}/pc-prepare-device.snap" "UqFziVZDHLSyO3TqSWgNBoAdHbLI4dAH"
+
+  for snap in "${extra_snaps}"/snapd*.snap; do
+    "${TESTSTOOLS}"/store-state add-to-channel "${NESTED_FAKESTORE_BLOB_DIR}" "${snap}" latest/stable
+  done
+  "${TESTSTOOLS}"/store-state add-to-channel "${NESTED_FAKESTORE_BLOB_DIR}" "${extra_snaps}/core${VERSION}.snap" latest/edge
+  "${TESTSTOOLS}"/store-state add-to-channel "${NESTED_FAKESTORE_BLOB_DIR}" "${pc_kernel_orig}" "${VERSION}/edge"
+  "${TESTSTOOLS}"/store-state add-to-channel "${NESTED_FAKESTORE_BLOB_DIR}" "${extra_snaps}/pc-kernel-oem.snap" "${VERSION}-oem/stable"
+  "${TESTSTOOLS}"/store-state add-to-channel "${NESTED_FAKESTORE_BLOB_DIR}" "${extra_snaps}/pc-prepare-device.snap" "${VERSION}/edge"
+
+  getassert() {
+    FILENAME=$1
+    ID=$2
+    SUM="$(snap info --verbose "$(realpath "${FILENAME}")" | sed '/^sha3-384: */{;s///;q;};d')"
+    cat "${TESTSLIB}/assertions/developer1.account-key"
+    echo
+    SNAPPY_FORCE_SAS_URL="${NESTED_UBUNTU_IMAGE_SNAPPY_FORCE_SAS_URL}" snap known --remote snap-declaration snap-id="${ID}" series=16
+    echo
+    SNAPPY_FORCE_SAS_URL="${NESTED_UBUNTU_IMAGE_SNAPPY_FORCE_SAS_URL}" snap known --remote snap-revision snap-sha3-384="${SUM}"
+  }
+
+  getassert "${pc_kernel_orig}" "pYVQrBcKmBa0mZ4CCN7ExT6jH8rY1hza" >pc-kernel.assert
+  getassert "${extra_snaps}/pc-prepare-device.snap" "UqFziVZDHLSyO3TqSWgNBoAdHbLI4dAH" >pc.assert
+
+  gendeveloper1 sign-model <"${TESTSLIB}/assertions/developer1-${VERSION}-classic-dangerous.json" >"${TESTSLIB}/assertions/developer1-${VERSION}-classic-dangerous.model"
+  gendeveloper1 sign-model <"${TESTSLIB}/assertions/developer1-${VERSION}-classic-dangerous-rev1.json" >"${TESTSLIB}/assertions/developer1-${VERSION}-classic-dangerous-rev1.model"
+
+  HYBRID_SYSTEM_MK_ROOT_FS="${PWD}/prepare-rootfs.sh" \
+  SNAPPY_FORCE_SAS_URL="${NESTED_UBUNTU_IMAGE_SNAPPY_FORCE_SAS_URL}" \
   "${TESTSTOOLS}"/setup_nested_hybrid_system.sh \
-     --model "${TESTSLIB}/assertions/classic-model-${VERSION}.assert" \
-     --gadget pc.snap \
+     --model "${TESTSLIB}/assertions/developer1-${VERSION}-classic-dangerous.model" \
+     --gadget "${extra_snaps}/pc-prepare-device.snap" \
      --gadget-assertion pc.assert \
-     --kernel pc-kernel.snap \
+     --kernel "${pc_kernel_orig}" \
      --kernel-assertion pc-kernel.assert
 
+  systemd-run --collect --unit fakedevicesvc fakedevicesvc localhost:11029
+
 restore: |
+  systemctl stop fakedevicesvc.service || true
+  "${TESTSTOOLS}/store-state" teardown-fake-store "${NESTED_FAKESTORE_BLOB_DIR}"
   tests.systemd stop-unit --remove "nested-vm"
 
+debug: |
+  journalctl -u fakestore.service
+  journalctl -u snapd.service
+
 execute: |
   remote.wait-for device-initialized
 
@@ -58,10 +163,10 @@ execute: |
 
   VERSION="$(tests.nested show version)"
 
-  remote.push "${TESTSLIB}/assertions/classic-model-rev1-${VERSION}.assert"
+  remote.push "${TESTSLIB}/assertions/developer1-${VERSION}-classic-dangerous-rev1.model"
 
   # remodel and reboot. we need to reboot because we swapped the kernel snap
-  change_id=$(remote.exec "sudo snap remodel --no-wait classic-model-rev1-${VERSION}.assert")
+  change_id=$(remote.exec "sudo snap remodel --no-wait developer1-${VERSION}-classic-dangerous-rev1.model")
   retry -n 100 --wait 5 remote.exec "snap tasks ${change_id} | grep 'INFO Task set to wait until a system restart allows to continue'"
   remote.exec 'test -f /run/reboot-required'
 
@@ -72,14 +177,12 @@ execute: |
   remote.exec 'snap changes' | MATCH "${change_id}\s+Done.+Refresh model assertion from revision 0 to 1"
 
   remote.exec "snap list" | MATCH "jq-core${VERSION}"
-  
-  # for now different tracks are used, but would nice to streamline
-  # this going forward
-  if [ "$VERSION" == 22 ]; then
-    remote.exec 'snap list pc-kernel' | awk 'NR != 1 { print $4 }' | MATCH "${VERSION}-oem/stable"
-  elif [ "$VERSION" == 24 ]; then
-    remote.exec 'snap list pc-kernel' | awk 'NR != 1 { print $4 }' | MATCH "${VERSION}-hwe/stable"
-  fi
+
+  case "$VERSION" in
+    22|24)
+      remote.exec 'snap list pc-kernel' | awk 'NR != 1 { print $4 }' | MATCH "${VERSION}-oem/stable"
+      ;;
+  esac
 
   # TODO: once we support installing recovery systems during the hybrid install,
   # we should switch this test to use a gadget that does not use "system-seed-null"