Merge branch 'sysext-openqa' into 'main'

Draft: Run minimal openQA tests

See merge request GNOME/gnome-shell!3391

.gitlab-ci.yml

@@ -1,5 +1,7 @@
 include:
   - remote: 'https://gitlab.gnome.org/GNOME/citemplates/raw/HEAD/flatpak/flatpak_ci_initiative.yml'
+  - remote: 'https://gitlab.gnome.org/tchx84/citemplates/raw/sysext/sysext/sysext_ci_initiative.yml'
+  - remote: 'https://gitlab.gnome.org/tchx84/citemplates/raw/sysext-openqa/sysext/sysext_openqa_ci_initiative.yml'
   - remote: 'https://gitlab.freedesktop.org/freedesktop/ci-templates/-/raw/057b052e682d8e5a20c1eb2dd60d5b87d2b56856/templates/fedora.yml'
   - remote: 'https://gitlab.freedesktop.org/freedesktop/ci-templates/-/raw/bc70242ffb8402243e934659ecc1a2d1c89eca2b/templates/ci-fairy.yml'
   - project: 'Infrastructure/openshift-images/gnome-release-service'
@@ -250,6 +252,26 @@ test-coverity:
     paths:
       - coverity
 
+sysext:
+  stage: build
+  variables:
+    SYSEXT_MESON_OPTIONS:
+      --werror
+      --fatal-meson-warnings
+      -Dextensions_app=false
+      -Dextensions-tool:bash_completion=disabled
+  before_script:
+    - .gitlab-ci/install-sysext-dependencies.sh
+  extends: .sysext.meson-build
+
+sysext_openqa:
+  extends: ".sysext_openqa"
+  needs: [sysext]
+  dependencies: [sysext]
+  stage: test
+  variables:
+    TESTS_SUBDIR: "tests/openqa"
+
 flatpak:
   stage: build
   needs: ["check_commit_log"]

.gitlab-ci/checkout-mutter.sh

@@ -8,9 +8,14 @@ fetch() {
 }
 
 mutter_target=
+mutter_repo="https://gitlab.gnome.org/GNOME/mutter.git"
 
-echo -n Cloning into mutter ...
-if git clone --quiet --depth=1 https://gitlab.gnome.org/GNOME/mutter.git; then
+if [[ "$CI_SERVER_URL" && "$CI_PROJECT_ROOT_NAMESPACE" ]]; then
+  mutter_repo="$CI_SERVER_URL/$CI_PROJECT_ROOT_NAMESPACE/mutter.git"
+fi
+
+echo -n Cloning into mutter from $mutter_repo...
+if git clone --quiet --depth=1 $mutter_repo; then
   echo \ done
 else
   echo \ failed

.gitlab-ci/install-sysext-dependencies.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# This script ensures that all dependencies required to build and run
+# a GNOME Shell system extension are present in:
+#
+# The building environment, by installing these dependencies to the container.
+# The running environment, by bundling these dependencies with the extension.
+#
+
+set -e
+
+# Install dependencies to $SYSEXT_DEST_DIR to bundle these with the extension.
+
+if [ -z ${SYSEXT_DEST_DIR+x} ]; then
+  echo "SYSEXT_DEST_DIR is not defined"
+  exit 1
+fi
+
+# Ensure that we're building against (and bundling) the right mutter branch
+# and its dependencies:
+
+SCRIPT_DIR="$(dirname $0)"
+
+$SCRIPT_DIR/checkout-mutter.sh
+./mutter/.gitlab-ci/install-sysext-dependencies.sh
+
+meson setup mutter/build mutter --prefix=/usr --libdir="lib/$(gcc -print-multiarch)"
+meson compile -C mutter/build
+meson install -C mutter/build --destdir $SYSEXT_DEST_DIR
+
+sudo meson install -C mutter/build
+
+# Ensure that any other dependency missing in GNOME OS is installed and bundled
+# here as it was done with mutter:

tests/openqa/config/scenario_definitions.yaml

@@ -0,0 +1,41 @@
+# This is the toplevel configuration for the OpenQA scenarios
+
+products:
+  gnomeos:
+    distri: gnomeos
+    flavor: iso
+    version: "master"
+    arch: "x86_64"
+
+machines:
+  qemu_x86_64:
+    backend: "qemu"
+    settings:
+      CDMODEL: scsi-cd
+      DESKTOP: gnomeos
+      HDDMODEL: virtio-blk
+      HDDSIZEGB: '40'
+      NICMAC: 52:54:00:12:34:56
+      NICMODEL: virtio-net
+      NICTYPE: user
+      NICVLAN: '0'
+      NUMDISKS: '1'
+      PART_TABLE_TYPE: gpt
+      QEMUCPU: host
+      QEMUCPUS: '2'
+      QEMUPORT: '20012'
+      QEMURAM: '2560'
+      QEMU_SMBIOS: 'type=11,path=/tests/config/smbios.txt'
+      QEMU_VIDEO_DEVICE: virtio-vga
+      UEFI: '1'
+      UEFI_PFLASH_CODE: /usr/share/qemu/ovmf-x86_64-code.bin
+      UEFI_PFLASH_VARS: /usr/share/qemu/ovmf-x86_64-vars.bin
+      VIRTIO_CONSOLE: '1'
+      VNC_TYPING_LIMIT: '5'
+
+job_templates:
+  gnome_apps:
+    product: gnomeos
+    machine: qemu_x86_64
+    settings:
+      HDD_1: /disk.img

tests/openqa/config/smbios.txt

@@ -0,0 +1 @@
+io.systemd.stub.kernel-cmdline-extra=console=ttyS0 systemd.journald.forward_to_console=1

tests/openqa/lib/gnomeosdistribution.pm

@@ -0,0 +1,45 @@
+# Based on https://github.com/os-autoinst/os-autoinst-distri-opensuse/blob/master/lib/susedistribution.pm
+
+package gnomeosdistribution;
+use base 'distribution';
+use strict;
+use warnings;
+use serial_terminal ();
+use testapi qw(diag);
+
+=head2 init
+
+Initialize the GNOME OS distribution-specific helpers.
+=cut
+
+sub init {
+    my ($self) = @_;
+
+    $self->SUPER::init();
+
+    $self->add_console('x11', 'tty-console', {tty => 2});
+    $self->add_console('gdm', 'tty-console', {tty => 1});
+    $self->add_console('user-virtio-terminal', 'virtio-terminal', {});
+}
+
+=head2 activate_console
+  activate_console($console)
+Callback whenever a console is selected for the first time. Accepts arguments
+provided to select_console().
+=cut
+
+sub activate_console {
+    my ($self, $console, %args) = @_;
+
+    if ($console eq 'user-virtio-terminal') {
+        my $user = $testapi::username;
+        diag "activate_console, console: $console, user: $user";
+
+        $self->{serial_term_prompt} = $user eq 'root' ? '# ' : '> ';
+        serial_terminal::login($user, $self->{serial_term_prompt});
+    } else {
+        diag 'activate_console called with unknown type, no action';
+    }
+}
+
+1;

tests/openqa/lib/serial_terminal.pm

@@ -0,0 +1,116 @@
+# GNOME OS openQA tests
+#
+# Based on SUSE's openQA tests
+#
+# See: https://github.com/os-autoinst/os-autoinst-distri-opensuse/blob/master/lib/serial_terminal.pm
+#
+# Copyright 2016-2019 SUSE LLC
+# SPDX-License-Identifier: FSFAP
+package serial_terminal;
+use 5.018;
+use warnings;
+use testapi;
+use autotest;
+use base 'Exporter';
+use Exporter;
+use Mojo::Util qw(b64_encode b64_decode sha1_sum trim);
+use Mojo::File 'path';
+use File::Basename;
+use File::Temp 'tempfile';
+
+BEGIN {
+    our @EXPORT = qw(
+      get_login_message
+      login
+      set_serial_prompt
+      serial_term_prompt
+    );
+}
+
+our $serial_term_prompt;
+
+=head2 get_login_message
+
+   get_login_message();
+
+Get login message printed by OS at the end of the boot.
+Suitable for testing whether boot has been finished:
+
+wait_serial(get_login_message(), 300);
+=cut
+
+sub get_login_message {
+    return qr/'GNOME OS Nightly.*/;
+}
+
+=head2 set_serial_prompt
+
+   set_serial_prompt($user);
+
+Set serial terminal prompt to given string.
+
+=cut
+
+sub set_serial_prompt {
+    $serial_term_prompt = shift // '';
+
+    die "Invalid prompt string '$serial_term_prompt'"
+      unless $serial_term_prompt =~ s/\s*$//r;
+    enter_cmd(qq/PS1="$serial_term_prompt"/);
+    wait_serial(qr/PS1="$serial_term_prompt"/);
+}
+
+=head2 login
+
+   login($user);
+
+Enters root's name and password to login. Also sets the prompt to something static without ANSI
+escape sequences (i.e. a single #) and changes the terminal width.
+
+=cut
+
+sub login {
+    die 'Login expects two arguments' unless @_ == 2;
+    my $user = shift;
+    my $prompt = shift;
+    my $escseq = qr/(\e [\(\[] [\d\w]{1,2})/x;
+
+    # Eat stale buffer contents, otherwise the code below may get confused
+    # after reboot and start typing the username before the console is actually
+    # ready to accept it
+    wait_serial(qr/login:\s*$/i, timeout => 3, quiet => 1);
+    # newline nudges the guest to display the login prompt, if this behaviour
+    # changes then remove it
+    send_key 'ret';
+    die 'Failed to wait for login prompt' unless wait_serial(qr/login:\s*$/i);
+    enter_cmd("$user");
+
+    my $re = qr/$user/i;
+    if (!wait_serial($re, timeout => 3)) {
+        record_info('RELOGIN', 'Need to retry login to workaround virtio console race', result => 'softfail');
+        enter_cmd("$user");
+        die 'Failed to wait for password prompt' unless wait_serial($re, timeout => 3);
+    }
+
+    if (length $testapi::password) {
+        die 'Failed to wait for password prompt' unless wait_serial(qr/Password:\s*$/i, timeout => 30);
+        type_password;
+        send_key 'ret';
+    }
+
+    die 'Failed to confirm that login was successful' unless wait_serial(qr/$escseq* \[\w+\@\w+\s~\]\$ $escseq* \s*$/x);
+
+    # Some (older) versions of bash don't take changes to the terminal during runtime into account. Re-exec it.
+    enter_cmd('export TERM=dumb; stty cols 2048; exec $SHELL');
+    die 'Failed to confirm that shell re-exec was successful' unless wait_serial(qr/$escseq* \[\w+\@\w+\s~\]\$ $escseq* \s*$/x);
+    set_serial_prompt($prompt);
+    # TODO: Send 'tput rmam' instead/also
+    assert_script_run('export TERM=dumb');
+    assert_script_run('echo Logged into $(tty)', timeout => 30, result_title => 'vconsole_login');
+}
+
+sub serial_term_prompt {
+    return $serial_term_prompt;
+}
+
+1;

tests/openqa/main.pm

@@ -0,0 +1,27 @@
+use strict;
+use warnings;
+use testapi;
+use autotest;
+use needle;
+use File::Basename;
+
+my $distri = testapi::get_required_var('CASEDIR') . '/lib/gnomeosdistribution.pm';
+require $distri;
+testapi::set_distribution(gnomeosdistribution->new);
+
+$testapi::username = 'testuser';
+$testapi::password = 'testingtesting123';
+
+my $testsuite = testapi::get_required_var('TEST');
+
+if ($testsuite eq "gnome_apps") {
+    $testapi::form_factor_postfix = '';
+    autotest::loadtest("tests/gnome_welcome.pm");
+    autotest::loadtest("tests/gnome_disable_update_notification.pm");
+    autotest::loadtest("tests/gnome_desktop.pm");
+
+} else {
+    die("Invalid testsuite: '$testsuite'");
+}
+
+1;

tests/openqa/needles/gnome_desktop_desktop.json

@@ -0,0 +1,36 @@
+{
+  "area": [
+    {
+      "width": 1022,
+      "height": 766,
+      "type": "match",
+      "ypos": 2,
+      "xpos": 2
+    },
+    {
+      "width": 228,
+      "height": 27,
+      "type": "exclude",
+      "ypos": 3,
+      "xpos": 407
+    },
+    {
+      "ypos": 123,
+      "type": "exclude",
+      "xpos": 154,
+      "height": 513,
+      "width": 716
+    },
+    {
+      "xpos": 945,
+      "type": "exclude",
+      "ypos": 140,
+      "height": 481,
+      "width": 79
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_desktop_desktop"
+  ]
+}

tests/openqa/needles/gnome_desktop_tour.json

@@ -0,0 +1,26 @@
+{
+  "area": [
+    {
+      "height": 20,
+      "ypos": 596,
+      "click_point": {
+        "ypos": 15,
+        "xpos": 52.5
+      },
+      "type": "match",
+      "width": 38,
+      "xpos": 382
+    },
+    {
+      "xpos": 294,
+      "width": 439,
+      "type": "match",
+      "height": 484,
+      "ypos": 141
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_desktop_tour"
+  ]
+}

tests/openqa/needles/gnome_firstboot_aboutyou_1.json

@@ -0,0 +1,28 @@
+{
+  "area": [
+    {
+      "xpos": 436,
+      "ypos": 270,
+      "width": 154,
+      "height": 39,
+      "type": "match",
+      "match": 95
+    },
+    {
+      "xpos": 950,
+      "ypos": 38,
+      "width": 67,
+      "height": 36,
+      "type": "match",
+      "match": 95,
+      "click_point": {
+        "xpos": 32,
+        "ypos": 18
+      }
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_firstboot_aboutyou_1"
+  ]
+}

tests/openqa/needles/gnome_firstboot_aboutyou_2.json

@@ -0,0 +1,26 @@
+{
+  "area": [
+    {
+      "type": "match",
+      "xpos": 951,
+      "ypos": 38,
+      "click_point": {
+        "ypos": 18,
+        "xpos": 33.5
+      },
+      "height": 36,
+      "width": 67
+    },
+    {
+      "height": 734,
+      "width": 1023,
+      "type": "match",
+      "xpos": 1,
+      "ypos": 32
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_firstboot_aboutyou_2"
+  ]
+}

tests/openqa/needles/gnome_firstboot_complete.json

@@ -0,0 +1,26 @@
+{
+  "area": [
+    {
+      "width": 171,
+      "click_point": {
+        "xpos": 33.5,
+        "ypos": 18
+      },
+      "xpos": 429,
+      "height": 31,
+      "ypos": 539,
+      "type": "match"
+    },
+    {
+      "type": "match",
+      "height": 734,
+      "ypos": 32,
+      "xpos": 1,
+      "width": 1022
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_firstboot_complete"
+  ]
+}

tests/openqa/needles/gnome_firstboot_language.json

@@ -0,0 +1,33 @@
+{
+  "area": [
+    {
+      "click_point": {
+        "ypos": 18,
+        "xpos": 33.5
+      },
+      "height": 36,
+      "width": 67,
+      "type": "match",
+      "ypos": 38,
+      "xpos": 951
+    },
+    {
+      "type": "match",
+      "ypos": 32,
+      "xpos": 1,
+      "height": 734,
+      "width": 1023
+    },
+    {
+      "xpos": 221,
+      "ypos": 247,
+      "width": 580,
+      "height": 370,
+      "type": "exclude"
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_firstboot_language"
+  ]
+}

tests/openqa/needles/gnome_firstboot_password_1.json

@@ -0,0 +1,15 @@
+{
+  "area": [
+    {
+      "xpos": 1,
+      "ypos": 33,
+      "type": "match",
+      "height": 732,
+      "width": 1022
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_firstboot_password_1"
+  ]
+}

tests/openqa/needles/gnome_firstboot_password_2.json

@@ -0,0 +1,26 @@
+{
+  "area": [
+    {
+      "ypos": 38,
+      "xpos": 951,
+      "type": "match",
+      "click_point": {
+        "ypos": 18,
+        "xpos": 33.5
+      },
+      "height": 36,
+      "width": 67
+    },
+    {
+      "type": "match",
+      "xpos": 0,
+      "ypos": 33,
+      "height": 700,
+      "width": 1022
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_firstboot_password_2"
+  ]
+}

tests/openqa/needles/gnome_firstboot_privacy.json

@@ -0,0 +1,26 @@
+{
+  "area": [
+    {
+      "click_point": {
+        "xpos": 33.5,
+        "ypos": 18
+      },
+      "height": 36,
+      "width": 67,
+      "type": "match",
+      "ypos": 38,
+      "xpos": 951
+    },
+    {
+      "width": 1022,
+      "height": 731,
+      "ypos": 32,
+      "xpos": 1,
+      "type": "match"
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_firstboot_privacy"
+  ]
+}

tests/openqa/needles/gnome_firstboot_timezone_1.json

@@ -0,0 +1,27 @@
+{
+  "area": [
+    {
+      "width": 90,
+      "click_point": {
+        "ypos": 10,
+        "xpos": 10,
+        "id": "search_entry"
+      },
+      "height": 20,
+      "type": "match",
+      "xpos": 260,
+      "ypos": 188
+    },
+    {
+      "height": 735,
+      "width": 1024,
+      "ypos": 33,
+      "xpos": 0,
+      "type": "match"
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_firstboot_timezone_1"
+  ]
+}

tests/openqa/needles/gnome_firstboot_timezone_2.json

@@ -0,0 +1,26 @@
+{
+  "area": [
+    {
+      "click_point": {
+        "xpos": 33.5,
+        "ypos": 18
+      },
+      "width": 335,
+      "height": 40,
+      "type": "match",
+      "ypos": 220,
+      "xpos": 264
+    },
+    {
+      "height": 737,
+      "width": 1023,
+      "type": "match",
+      "xpos": 1,
+      "ypos": 31
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_firstboot_timezone_2"
+  ]
+}

tests/openqa/needles/gnome_firstboot_welcome.json

@@ -0,0 +1,33 @@
+{
+  "area": [
+    {
+      "width": 67,
+      "click_point": {
+        "xpos": 33.5,
+        "ypos": 18
+      },
+      "height": 36,
+      "type": "match",
+      "ypos": 38,
+      "xpos": 951
+    },
+    {
+      "width": 1024,
+      "height": 736,
+      "xpos": 0,
+      "ypos": 32,
+      "type": "match"
+    },
+    {
+      "type": "exclude",
+      "xpos": 10,
+      "ypos": 100,
+      "width": 1010,
+      "height": 56
+    }
+  ],
+  "properties": [],
+  "tags": [
+    "gnome_firstboot_welcome"
+  ]
+}

tests/openqa/needles/gnome_firstboot_welcome_lang.json

@@ -0,0 +1,59 @@
+{
+  "area": [
+    {
+      "xpos": 225,
+      "ypos": 204,
+      "width": 578,
+      "height": 30,
+      "type": "match",
+      "match": 93,
+      "click_point": {
+        "xpos": 289,
+        "ypos": 15,
+        "id": "search"
+      }
+    },
+    {
+      "xpos": 224,
+      "ypos": 488,
+      "width": 576,
+      "height": 52,
+      "type": "match",
+      "match": 93,
+      "click_point": {
+        "xpos": 288,
+        "ypos": 26,
+        "id": "lang_arabic"
+      }
+    },
+    {
+      "xpos": 224,
+      "ypos": 537,
+      "width": 574,
+      "height": 57,
+      "type": "match",
+      "match": 93,
+      "click_point": {
+        "xpos": 287,
+        "ypos": 28.5,
+        "id": "lang_japanese"
+      }
+    },
+    {
+      "xpos": 222,
+      "ypos": 443,
+      "width": 581,
+      "height": 51,
+      "type": "match",
+      "match": 93,
+      "click_point": {
+        "xpos": 290.5,
+        "ypos": 20.5,
+        "id": "lang_russian"
+      }
+    }
+  ],
+  "tags": [
+    "gnome_firstboot_welcome_lang"
+  ]
+}

tests/openqa/tests/gnome_desktop.pm

@@ -0,0 +1,27 @@
+use base 'basetest';
+use strict;
+use testapi;
+
+sub run {
+    my $self = shift;
+
+    assert_and_click('gnome_desktop_tour', timeout => 120, button => 'left');
+    assert_screen('gnome_desktop_desktop', 20);
+}
+
+sub post_fail_hook {
+    # When GDM, gnome-session or gnome-shell have problems, its usually
+    # this test that fails. Having a full journal is very important for
+    # debugging this kind of thing.
+
+    select_console('user-virtio-terminal');
+    assert_script_run('journalctl --merge --output export | xz > /tmp/journal.xz; chmod 777 /tmp/journal.xz');
+    upload_asset('/tmp/journal.xz');
+    select_console('x11');
+}
+
+sub test_flags {
+    return { fatal => 1 };
+}
+
+1;

tests/openqa/tests/gnome_disable_update_notification.pm

@@ -0,0 +1,23 @@
+use base 'basetest';
+use strict;
+use testapi;
+
+sub run {
+    my $self = shift;
+
+    # App tests fail if the "Software Updates Ready to Install" notification
+    # appears over the top.
+    select_console('user-virtio-terminal');
+    assert_script_run('gsettings set org.gnome.desktop.notifications.application:/org/gnome/desktop/notifications/application/org-gnome-software/ enable false');
+    assert_script_run('cat /proc/cmdline');
+    assert_script_run('ls -la /dev/');
+    assert_script_run('ls -la /var/lib/extensions/extension/usr/bin/');
+    assert_script_run('ls -la /usr/lib/x86_64-linux-gnu/libwayland-egl.so.1.23.0');
+    assert_script_run('ls -la /usr/bin/mutter');
+    assert_script_run('ls -la /usr/bin/gnome-shell');
+    assert_script_run('ls -la /usr/lib/extension-release.d/');
+
+    select_console('x11');
+}
+
+1;

tests/openqa/tests/gnome_welcome.pm

@@ -0,0 +1,52 @@
+use base 'basetest';
+use strict;
+use testapi;
+
+# Taken from https://github.com/os-autoinst/os-autoinst-distri-opensuse/blob/master/lib/utils.pm#L110
+use constant SLOW_TYPING_SPEED => 13;
+
+sub run {
+    my $self = shift;
+
+    # Wait for machine to boot
+    assert_and_click('gnome_firstboot_welcome', timeout => 600, button => 'left');
+
+    # Language and privacy settings
+    assert_and_click('gnome_firstboot_language', timeout => 10, button => 'left');
+    assert_and_click('gnome_firstboot_privacy', timeout => 10, button => 'left');
+
+    # Timezone selection
+    assert_screen('gnome_firstboot_timezone_1', 30);
+    # The text entry field should be focused by default when we reach the timezone panel,
+    # but this randomly doesn't work. Workaround is to click on the widget before typing.
+    # See: <https://gitlab.gnome.org/GNOME/gnome-initial-setup/-/issues/156>.
+    click_lastmatch(point_id => "search_entry");
+    type_string('London, East', wait_screen_change => 6, max_interval => SLOW_TYPING_SPEED);
+    assert_and_click('gnome_firstboot_timezone_2', timeout => 20, button => 'left');
+    # We need to move focus to the next button, so we use tab and once the button is in focus, then enter to click it.
+    send_key('tab');
+    send_key('tab');
+    send_key('ret');
+
+    # User setup
+    assert_screen('gnome_firstboot_aboutyou_1', 10);
+    type_string($testapi::username);
+    assert_and_click('gnome_firstboot_aboutyou_2', timeout => 10, button => 'left');
+    assert_screen('gnome_firstboot_password_1', 10);
+    type_string($testapi::password);
+    send_key('tab');
+    send_key('tab');
+    type_string($testapi::password);
+    assert_and_click('gnome_firstboot_password_2', timeout => 10, button => 'left');
+
+    # Complete
+    assert_and_click('gnome_firstboot_complete', timeout => 10, button => 'left');
+    wait_still_screen(1);
+    save_screenshot;
+}
+
+sub test_flags {
+    return { fatal => 1 };
+}
+
+1;