Issue #11455: Split flatpak jobs

- Dependencies not present in GNOME runtime are now built in deps stage
  This makes easier to follow the progress of the overall pipeline and
  to know how much time was spent on each stage
  (like crossbuilds, the artifact size is brutal but only lasts 2 hours)

- babl and GEGL build now have output in GitLab runner, unlike others deps
  This makes clearer to spot if something goes wrong in these crucial deps

- GIMP is still built in its stage but now alone, like Debian and Windows
  This makes possible to retrigger only this job when runner errors occours,
  without needing to start monolithically the deps+gimp build from scratch

Also:

- babl, GEGL and GIMP now have meson-log.txt artifact like other builds

- dist job now have all commands self-contained on its script (needs to
  be merged to be tested according to my tests)

.gitlab-ci.yml

@@ -332,41 +332,59 @@ gimp-debian-x64:
       junit: "_build${ARTIFACTS_SUFFIX}/meson-logs/testlog.junit.xml"
     expire_in: 2 days
 
-gimp-flatpak-x64:
+.flatpak-x64:
   extends: .default
   rules:
     - if: '$CI_MERGE_REQUEST_LABELS =~ /.*5. Flatpak package.*/'
       interruptible: true
     - if: '$GIMP_CI_FLATPAK != null'
-  needs: []
-  stage: gimp
   tags:
     - flatpak
   image: 'quay.io/gnome_infrastructure/gnome-runtime-images:gnome-master'
-  # FIXME: Flatpak design mandates to build everything from scratch on CI
+
+deps-flatpak-x64:
+  extends: .flatpak-x64
+  rules:
+    - !reference [.flatpak-x64, rules]
+  needs: []
+  stage: dependencies
+  # FIXME: GitLab CI caching does NOT work with flatpak
   # See: https://gitlab.gnome.org/Infrastructure/Infrastructure/-/issues/1502
   cache: []
     # key: ${CI_JOB_NAME}-pkg
     # paths:
-        # .flatpak-builder/downloads
+        # .flatpak-builder/
-        # .flatpak-builder/git
+    # key: $CI_JOB_NAME
-    # !reference [.default, cache]
+    # paths:
+        # _ccache/
   script:
-    # FIXME: Flatpak design mandates to build monolithically (aka in one job) on CI
     - bash build/linux/flatpak/1_build-deps-flatpak.sh
-    - bash build/linux/flatpak/2_build-gimp-flatpak.sh
-    # FIXME: Flatpak design mandates to run in the same CI job even the dist commands!
-    - bash build/linux/flatpak/4_dist-gimp-flatpak.sh
   artifacts:
     paths:
+      - .flatpak-builder.tar
       - flatpak-builder.log
-      - gimp-git.flatpak
+      - babl-meson-log.tar
+      - gegl-meson-log.tar
+    expire_in: 2 hours
+
+gimp-flatpak-x64:
+  extends: .flatpak-x64
+  rules:
+    - !reference [.flatpak-x64, rules]
+  needs: ["deps-flatpak-x64"]
+  stage: gimp
+  # FIXME: GitLab CI caching does NOT work with flatpak
+  # See: https://gitlab.gnome.org/Infrastructure/Infrastructure/-/issues/1502
+  cache: []
+    # key: $CI_JOB_NAME
+    # paths:
+        # _ccache/
+  script:
+    - bash build/linux/flatpak/2_build-gimp-flatpak.sh
+  artifacts:
+    paths:
       - repo.tar
-      # FIXME: these can't be fetched, even if we bypass the symlink (module -> module-1)
+      - gimp-meson-log.tar
-      #        maybe also due to flatpak design?
-      # .flatpak-builder/build/babl-1/_flatpak_build/meson-logs/meson-log.txt
-      # .flatpak-builder/build/gegl-1/_flatpak_build/meson-logs/meson-log.txt
-      # .flatpak-builder/build/gimp-1/_flatpak_build/meson-logs/meson-log.txt
     expire_in: 2 days
 
 ## WINDOWS 64-bit CI (cross-build crossroad) ##
@@ -757,10 +775,7 @@ dist-flatpak-weekly:
   stage: distribution
   cache: []
   script:
-    - if [ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]; then
+    - bash build/linux/flatpak/4_dist-gimp-flatpak.sh
-        exit 0;
-      fi
-    - !reference [.publish_nightly, script]
   artifacts:
     expose_as: 'Linux flatpak'
     paths:

build/linux/flatpak/1_build-deps-flatpak.sh

@@ -4,22 +4,6 @@
 ARCH=$(uname -m)
 
 
-if [ -z "$GITLAB_CI" ]; then
-  # Make the script work locally
-  if [ "$0" != 'build/linux/flatpak/1_build-deps-flatpak.sh' ]; then
-    echo 'To run this script locally, please do it from to the gimp git folder'
-    exit 1
-  fi
-  flatpak update -y
-  if [ -z "$GIMP_PREFIX" ]; then
-    export GIMP_PREFIX="$PWD/../_install-$ARCH"
-  fi
-  if [ ! -d "$GIMP_PREFIX" ]; then
-    mkdir -p "$GIMP_PREFIX"
-  fi
-fi
-
-
 # Install part of the deps
 if [ -z "$GITLAB_CI" ]; then
   flatpak remote-add --if-not-exists --user --from gnome-nightly https://nightly.gnome.org/gnome-nightly.flatpakrepo
@@ -30,8 +14,41 @@ flatpak install --user flathub org.freedesktop.Sdk.Extension.llvm18 -y
 
 
 # Clone and build the deps not present in GNOME runtime
-# (Flatpak design mandates to reinstall everything ('--force-clean') on the prefix)
+if [ -z "$GITLAB_CI" ] && [ "$1" != '--ci' ]; then
-if [ -z "$GITLAB_CI" ]; then
+  # Make the script work locally
+  if [ "$0" != 'build/linux/flatpak/1_build-deps-flatpak.sh' ]; then
+    echo 'To run this script locally, please do it from to the gimp git folder'
+    exit 1
+  fi
+  flatpak update -y
+  if [ -z "$GIMP_PREFIX" ]; then
+    export GIMP_PREFIX="$PWD/../_install-$ARCH"
+  fi
+  if [ ! -d "$GIMP_PREFIX" ]; then
+    mkdir -p "$GIMP_PREFIX"
+  fi
+
   flatpak-builder --force-clean --ccache --state-dir=../.flatpak-builder --keep-build-dirs --stop-at=gimp \
                   "$GIMP_PREFIX" build/linux/flatpak/org.gimp.GIMP-nightly.json 2>&1 | tee flatpak-builder.log
+
+
+elif [ "$GITLAB_CI" ] || [ "$1" = '--ci' ]; then
+  export GIMP_PREFIX="$PWD/_install-$ARCH"
+
+  # (The deps building is too long and no complete output would be collected,
+  # even from GitLab runner messages. So, let's silent and save logs as a file.)
+  echo 'Building dependencies not present in GNOME runtime'
+  flatpak-builder --force-clean --user --disable-rofiles-fuse --keep-build-dirs --build-only --stop-at=babl \
+                  "$GIMP_PREFIX" build/linux/flatpak/org.gimp.GIMP-nightly.json &> flatpak-builder.log
+
+  # Let's output at least babl and GEGL building
+  flatpak-builder --force-clean --user --disable-rofiles-fuse --keep-build-dirs --build-only --stop-at=gimp \
+                  "$GIMP_PREFIX" build/linux/flatpak/org.gimp.GIMP-nightly.json
+  if [ "$1" != '--ci' ]; then
+    tar cf babl-meson-log.tar .flatpak-builder/build/babl-1/_flatpak_build/meson-logs/meson-log.txt
+    tar cf gegl-meson-log.tar .flatpak-builder/build/gegl-1/_flatpak_build/meson-logs/meson-log.txt
+
+    # Save built deps for 'gimp-flatpak-x64' job
+    tar cf .flatpak-builder.tar .flatpak-builder/
+  fi
 fi

build/linux/flatpak/2_build-gimp-flatpak.sh

@@ -16,7 +16,6 @@ if [ -z "$GITLAB_CI" ] && [ "$1" != '--ci' ]; then
     export GIMP_PREFIX="$PWD/../_install-$ARCH"
   fi
 
-
   # Build GIMP only
   if [ ! -f "_build-$ARCH/build.ninja" ]; then
     mkdir -p _build-$ARCH && cd _build-$ARCH
@@ -34,14 +33,26 @@ if [ -z "$GITLAB_CI" ] && [ "$1" != '--ci' ]; then
 elif [ "$GITLAB_CI" ] || [ "$1" = '--ci' ]; then
   export GIMP_PREFIX="$PWD/_install-$ARCH"
 
+  if [ "$1" != '--ci' ]; then
+    # Extract deps from previous job
+    echo 'Extracting previously built dependencies'
+    tar xf .flatpak-builder.tar
+  fi
 
   # GNOME script to customize gimp module in the manifest (not needed)
   #rewrite-flatpak-manifest build/linux/flatpak/org.gimp.GIMP-nightly.json gimp ${CONFIG_OPTS}
 
+  # Build GIMP only
+  flatpak-builder --force-clean --user --disable-rofiles-fuse --keep-build-dirs --build-only \
+                  "$GIMP_PREFIX" build/linux/flatpak/org.gimp.GIMP-nightly.json
+  if [ "$1" != '--ci' ]; then
+    tar cf gimp-meson-log.tar .flatpak-builder/build/gimp-1/_flatpak_build/meson-logs/meson-log.txt
+  fi
 
-  # Clone and build the deps not present in GNOME runtime (and GIMP)
+  # Cleanup GIMP_PREFIX and export it to OSTree repo
-  # (The deps building is too long and no complete output would be collected,
+  flatpak-builder --user --disable-rofiles-fuse --finish-only --repo=repo \
-  # even from GitLab runner messages. So, let's silent and save logs as a file.)
+                  "$GIMP_PREFIX" build/linux/flatpak/org.gimp.GIMP-nightly.json
-  flatpak-builder --force-clean --user --disable-rofiles-fuse --repo=repo ${BRANCH:+--default-branch=$BRANCH} \
+  if [ "$1" != '--ci' ]; then
-                  "$GIMP_PREFIX" build/linux/flatpak/org.gimp.GIMP-nightly.json &> flatpak-builder.log
+    tar cf repo.tar repo/
+  fi
 fi

build/linux/flatpak/4_dist-gimp-flatpak.sh

@@ -1,21 +1,25 @@
 #!/bin/sh
 
-# This script should be used only by 'dist-flatpak-weekly' job but
-# uploading repo/ in previous 'gimp-flatpak-x64' isn't possible
-# Let's keep the script, anyway, for clarity about the dist procedure
 
-
+if [ "$GITLAB_CI" ]; then
-# Generate a Flatpak bundle to be tested with GNOME runtime installed
+  # Extract previously exported repo/
-# (it is NOT a "real"/full bundle, deps from GNOME runtime are not bundled)
+  tar xf repo.tar
-flatpak build-bundle repo gimp-git.flatpak --runtime-repo=https://nightly.gnome.org/gnome-nightly.flatpakrepo org.gimp.GIMP ${BRANCH}
-
-# Only distribute on GNOME nightly if from 'master' branch
-if [ "$CI_COMMIT_BRANCH" != "$CI_DEFAULT_BRANCH" ]; then
-  exit 0
 fi
 
-# Prepare OSTree repo for GNOME nightly distribution
-tar cf repo.tar repo/
 
-# The following commands we take directly from 'flatpak_ci_initiative.yml'
+# Generate a Flatpak "bundle" to be tested with GNOME runtime installed
-# See 'dist-flatpak-weekly' job
+# (it is NOT a real/full bundle, deps from GNOME runtime are not bundled)
+echo 'Packaging repo as .flatpak'
+flatpak build-bundle repo gimp-git.flatpak --runtime-repo=https://nightly.gnome.org/gnome-nightly.flatpakrepo org.gimp.GIMP ${BRANCH}
+
+
+# Publish GIMP repo in GNOME nightly
+# We take the commands from 'flatpak_ci_initiative.yml'
+if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then
+  curl https://gitlab.gnome.org/GNOME/citemplates/raw/master/flatpak/flatpak_ci_initiative.yml --output flatpak_ci_initiative.yml
+  IFS=$'\n' cmd_array=($(cat flatpak_ci_initiative.yml | sed -n '/flatpak build-update-repo/,/exit $result\"/p' | sed 's/    - //'))
+  IFS=$' \t\n'
+  for cmd in "${cmd_array[@]}"; do
+    eval "$cmd" || continue
+  done
+fi