sof/.github/workflows/zephyr.yml

---

name: Zephyr

# 'workflow_dispatch' allows running this workflow manually from the
# 'Actions' tab
# yamllint disable-line rule:truthy
on: [push, pull_request, workflow_dispatch, workflow_call]

# Specifies group name that stops previous wokrflows if the name matches
concurrency:
  # eg. "Zephyr-pull_request-my_fork_branch_to_merge"
  # eg. "Zephyr-push-refs/heads/my_branch_merging"
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

jobs:

  manifest-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          path: ./workspace/sof

      - name: plain west update
        run: |

          : This plain 'west update' does not provide 100% certainty that
          : all the manifest revisions make sense but it is quick and
          : will catch many revision problems. Other jobs typically
          : use 'west update --narrow' which is faster but
          : also able to fetch "wild" SHA1s from any random place! --narrow
          : is useful for testing unmerged Zephyr commits but risks
          : accepting "invalid" ones, this will not.

          pip3 install west
          cd workspace/sof/
          west init -l
          west update --fetch-opt=--filter=tree:0

      # Temporary check until we change west.yml and stop nesting rimage
      # and tomlc99 inside sof which will cleanly separate them from
      # XTOS submodules and... temporarily break every CI, which is why
      # it hasn't been done yet.
      - name: git submodules consistency
        run: |

          cd workspace/sof
          git submodule update --init --recursive

          west update
          if git status --porcelain=v2 | grep ^ ; then
             git status
             echo 'FAIL: inconsistency between git submodules and west.yml!'
             echo 'See rimage comment in west.yml.'
             echo 'Always use "git status"'
             exit 1
          fi


  # Temporary hacks to compile the very first LP64 configuration early
  # before it becomes supported by the regular
  # sof/scripts/xtensa-build-zephyr.py configuration script. Then this
  # job will be disappear, folded back in the regular build-* jobs below.
  LP64-WIP:
    runs-on: ubuntu-22.04

    steps:
      - uses: actions/checkout@v3
        with:
          path: ./workspace/sof

      - name: west clones
        run: pip3 install west && cd workspace/sof/ && west init -l &&
               west update --narrow --fetch-opt=--filter=tree:0

      # Not strictly necessary but saves a lot of scrolling in the next step
      # Caching a 12G image is unfortunately not possible:
      #   https://github.com/ScribeMD/docker-cache/issues/304
      # For faster builds we would have to pay for some persistent runners.
      - name: Download docker image && ls /opt/toolchains/
        run: cd workspace && ./sof/zephyr/docker-run.sh ls -l /opt/toolchains/

      - name: build with many warnings, work in progress
        run: |
          # One of the main problems: the old sof-logger is hardcoded to
          # four 32bits parameters.
          cd workspace && ./sof/zephyr/docker-run.sh /bin/sh -c \
             'ln -s  /opt/toolchains/zephyr-sdk-*  ~/;
              west build --board mimx93_evk_a55 sof/app'          


  build-linux:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        # These keys are kept short because Github's left column is not resizable.
        # Search "zephyr_revision" and see below what they expand to.
        zephyr_revision: [
          mnfst,  # special value: don't override sof/west.yml
          zmain,  # Zephyr's main branch
        ]
        # Using groups to avoid spamming the small results box with too
        # many lines. Pay attention to COMMAS.
        IPC_platforms: [
          # - IPC3 default
          imx8 imx8x imx8m,
          tgl tgl-h,  # UNSUPPORTED! Will be removed
          # - IPC4 default
          mtl, lnl,
          # Temporary testbed for Zephyr development.
          tgl tgl-h,
        ]
        build_opts: [""]
        # Sparse matrices are complicated, you must read this page slowly:
        # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
        include:
          # specify one extra -d combination without affecting the main matrix
          - build_opts: -d
            zephyr_revision: mnfst
            IPC_platforms: mtl

    steps:
      - uses: actions/checkout@v3
        # Download a full clone to fix `git describe`, sof_version.h and
        # build reproducibility. sof.git is still small.
        # This is especially useful for daily builds (but not just).
        with:
          fetch-depth: 0
          path: ./workspace/sof

      - name: west clones

        run: pip3 install west && cd workspace/sof/ && west init -l &&
               west update --narrow --fetch-opt=--filter=tree:0

      - name: select zephyr revision
        run: |
          cd workspace
          if [ 'mnfst' =  '${{ matrix.zephyr_revision }}' ]; then
             rem_rev=$(git -C zephyr rev-parse HEAD)
          else
             case '${{ matrix.zephyr_revision }}' in
               'zmain') rem_rev='main' ;;
               *)       echo 'Unknown matrix.zephyr_revision'; exit 1 ;;
             esac
             ( cd sof/submanifests/
               sed -e "s#=sof_zephyr_revision_override=#${rem_rev}#" \
                 sof-ci-jenkins/zephyr-override-template.yml > test-zephyr-main.yml
             )
             west update --narrow  --fetch-opt=--filter=tree:0
          fi

          # Because we used git tricks to speed things up, we now have two git
          # problems:
          #
          # 1. git fetch without a refspec fetches zero tag. So `git describe` is
          # different from everyone else. west normally adds `--tags` as a workaround
          # but this is inefficient because it fetches ALL tags; so `west --narrow`
          # does not use --tags.
          #
          # 2. west fetches using the remote URL, not the remote name. So remote
          # branches (if any) are missing from --decorate below
          #
          #  => an "empty" and quick fetch _with_ a refspec and the remote name fixes
          #     both issues in no time.

          cd zephyr
          git fetch --filter=tree:0 zephyrproject "$rem_rev":_branch_placeholder
          git branch -D _branch_placeholder

          set -x
          west list
          west status
          git   log --oneline -n 5 --decorate --graph --no-abbrev-commit          

      # Not strictly necessary but saves a lot of scrolling in the next step
      # Caching a 12G image is unfortunately not possible:
      #   https://github.com/ScribeMD/docker-cache/issues/304
      # For faster builds we would have to pay for some persistent runners.
      - name: Download docker image && ls /opt/toolchains/
        run: cd workspace && ./sof/zephyr/docker-run.sh ls -l /opt/toolchains/

      # https://github.com/zephyrproject-rtos/docker-image
      # Note: env variables can be passed to the container with
      # -e https_proxy=...
      - name: build
        run: cd workspace && ./sof/zephyr/docker-run.sh
             ./sof/zephyr/docker-build.sh --cmake-args=-DEXTRA_CFLAGS=-Werror
             --cmake-args=-DEXTRA_CXXFLAGS=-Werror
             --cmake-args=-DEXTRA_AFLAGS='-Werror -Wa,--fatal-warnings'
             --cmake-args=--warn-uninitialized
             ${{ matrix.build_opts }} ${{ matrix.IPC_platforms }}

      - name: Upload build artifacts
        uses: actions/upload-artifact@v3
        if: ${{ matrix.zephyr_revision == 'mnfst' }}
        with:
          name: linux-build ${{ matrix.build_opts }} ${{ matrix.IPC_platforms }}
          path: |
            ${{ github.workspace }}/workspace/build-sof-staging
            ${{ github.workspace }}/workspace/**/compile_commands.json            

  build-windows:
    runs-on: windows-latest
    strategy:
      fail-fast: false
      matrix:
        # Using groups to avoid spamming the small results box with too
        # many lines. Pay attention to COMMAS.
        platforms: [
          # - IPC3 default
          imx8 imx8x imx8m,
          tgl tgl-h,  # UNSUPPORTED! Will be removed
          # - IPC4 default
          mtl,
          # Very few IPC3 platforms support IPC4 too.
          tgl tgl-h,
        ]
        build_opts: [""]
        # Sparse matrices are complicated, see comments on Linux matrix above.
        include:
          - build_opts: -d
            platforms: mtl


    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
          path: ./workspace/sof

      # Cache artifacts so we do not overload external servers with downloads
      # Remember to change step key if you change the tools so old cache is not restored,
      # or delete cache manually in Github Actions tab
      - name: Cache unzip
        id: cache-unzip
        uses: actions/cache@v3.0.11
        with:
          path: unzip-5.51-1-bin.zip
          key: ${{ runner.os }}-cache-unzip-5-51-1

      - name: Cache wget
        id: cache-wget
        uses: actions/cache@v3.0.11
        with:
          path: wget-1.11.4-1-bin.zip
          key: ${{ runner.os }}-cache-wget-1-11-4-1

      # Keep this SDK version identical to the one in
      # sof/zephyr/docker-run.sh
      - name: Cache Zephyr SDK 0.16.1
        id: cache-zephyr-sdk
        uses: actions/cache@v3.0.11
        with:
          path: zephyr-sdk-0.16.1_windows-x86_64.7z
          key: ${{ runner.os }}-cache-zephyr-sdk-0-16-1

      # Wget is needed by Zephyr SDK setup.cmd installation script
      - name: Download wget
        if: ${{ steps.cache-wget.outputs.cache-hit != 'true' }}
        run: |
          curl -L -O http://downloads.sourceforge.net/gnuwin32/wget-1.11.4-1-bin.zip          

      - name: Download Zephyr SDK 0.16.1
        if: ${{ steps.cache-zephyr-sdk.outputs.cache-hit != 'true' }}
        run: |  # yamllint disable-line rule:line-length
          curl -L -O `
          https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v0.16.1/zephyr-sdk-0.16.1_windows-x86_64.7z

      # Unzips every .zip package to directory matching its name without extension
      - name: Unzip downloaded packages
        run: |
          7z x *.zip -o*
          7z x *.7z -o*          

      - name: Add unzip to system PATH
        run: |
          echo "${{ github.workspace }}/unzip-5.51-1-bin/bin" | `
          Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append          

      - name: Add wget to system PATH
        run: |
          echo "${{ github.workspace }}/wget-1.11.4-1-bin/bin" | `
          Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append          

      # Install Zephyr SDK - all toolchains including Host Tools
      # and registering CMake package in the registry
      # setup.cmd may not be called in from msys shell as it does not parse
      # forward slash script input arguments correctly.
      - name: Install Zephyr SDK
        run: zephyr-sdk-0.16.1_windows-x86_64/zephyr-sdk-0.16.1/setup.cmd /t all /h /c

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.8'

      - name: West install
        run: pip3 install west

      - name: West clone
        working-directory: ${{ github.workspace }}/workspace
        # Keep in sync with build-linux above
        run: |
          west init -l sof
          west update --narrow --fetch-opt=--filter=tree:0

          # Get some tags to fix `git describe`, see build-linux comments above.
          cd zephyr
          $_rev = "$(git rev-parse HEAD)"
          git fetch --filter=tree:0 zephyrproject "${_rev}:_branch_placeholder"
          git branch -D _branch_placeholder          


      # Call Setup Python again to save the PIP packages in cache
      - name: Setup Python
        uses: actions/setup-python@v4
        id: cache-python
        with:
          python-version: '3.8'
          cache: 'pip'
          cache-dependency-path: workspace/zephyr/scripts/requirements.txt

      # All requirements will be satisfied if the restored cache matches existing state
      - name: Validate python PIP cache
        working-directory: ${{ github.workspace }}/workspace
        run: pip install -r zephyr/scripts/requirements.txt

      # Ninja has been coming and going, see #8250
      - name: choco install ninja
        run: |
          choco install ninja
          ninja.exe --version          

      # MSYS2 provides gcc x64_86 toolchain & openssl
      # Installs in D:/a/_temp/msys64
      #
      # Note there is already C:/msys64/ provided by
      # https://github.com/actions/runner-images/blob/win22/20230918.1/images/win/Windows2022-Readme.md
      # Is it not good enough? Maybe it could save 20-30s.
      - name: Initialize MSYS2
        uses: msys2/setup-msys2@v2
        with:
          msystem: MSYS
          install: gcc openssl-devel
          path-type: inherit

      - name: Build
        shell: msys2 {0}
        working-directory: ${{ github.workspace }}/workspace
        run: python sof/scripts/xtensa-build-zephyr.py
          --no-interactive
          --cmake-args=-DEXTRA_CFLAGS=-Werror
          --cmake-args=-DEXTRA_CXXFLAGS=-Werror
          --cmake-args=-DEXTRA_AFLAGS='-Werror -Wa,--fatal-warnings'
          --cmake-args=--warn-uninitialized ${{ matrix.build_opts }} ${{ matrix.platforms }}

      - name: Upload build artifacts
        uses: actions/upload-artifact@v3
        with:
          name: windows-build ${{ matrix.build_opts }} ${{ matrix.platforms }}
          path: |
            ${{ github.workspace }}/workspace/build-sof-staging
            ${{ github.workspace }}/workspace/**/compile_commands.json            


  compare-linux-win:

    runs-on: ubuntu-latest

    # - We don't compare _all_ the builds, and
    # - even when some of the ones we compare fail, we still want to compare the rest.
    if: ${{ always() }}
    needs: [build-linux, build-windows]

    steps:
      - uses: actions/checkout@v3
        # we need only one script but it's simpler to get the (last
        # revision of the) whole repo and it takes seconds.
        with:
          # Isolate the clone in a subdirectory to make sure globbing
          # does not catch random SOF files.
          path: ./sof

      - name: Download Windows and Linux builds
        uses: actions/download-artifact@v3

      - name: apt-get dos2unix
        run: sudo apt-get update; sudo apt-get -y install dos2unix

      - name: Delete and fix expected differences
        run: |
          ls -l
          # run it twice to make sure it's idempotent
          for i in 0 1; do
            ./sof/zephyr/scripts/clean-expected-release-differences.sh \
               windows-build* linux-build*
          done          

      - name: Compare Linux vs Windows builds
        run: |
          diffs=0; ls -l

          # Check not too much is missing (it happened!)
          # http://mywiki.wooledge.org/ParsingLs
          for regdir in 'linux-build *mtl' \
                        'windows-build *mtl' \
                        'windows-build *tgl tgl-h'; do
            find . -maxdepth 1 | grep -q "\./${regdir}\$" ||
              { >&2 printf 'Missing %s\n' "${regdir}"; exit 1; }
          done

          set -x
          for windir in windows-build*; do
            lindir=linux-"${windir#windows-}"
            diff -qr "$lindir" "$windir" || : $((diffs++))
          done
          exit $diffs