Add versioning + CI to publish in Packman

2024-07-19 16:43:26 -07:00
parent a854375c1c
commit 32328d4e72
25 changed files with 399 additions and 8 deletions
@@ -0,0 +1,35 @@
+variables:
+  AWS_ACCESS_KEY_ID: $AWS_OMNIVERSEDOCS_ACCESS_KEY_ID
+  AWS_SECRET_ACCESS_KEY: $AWS_OMNIVERSEDOCS_SECRET_ACCESS_KEY
+  DXVK_REMIX_NVIDIA_MIN_DRIVER: 0
+
+stages:
+  - build_package
+  - publish_extensions
+  - publish_package
+  - publish_package_scheduled
+  - gitlab_release
+
+include:
+  # Job definitions
+  - local: tools/ci/gitlab/build/build_package.yml
+  - local: tools/ci/gitlab/gitlab/build_gitlab_package.yml
+  - local: tools/ci/gitlab/gitlab/publish_package.yml
+  - local: tools/ci/gitlab/gitlab/publish_release.yml
+  # Base definitions shared between jobs
+  - local: tools/ci/gitlab/defaults/defaults.yml
+  - local: tools/ci/gitlab/defaults/gitlab_release.yml
+  - local: tools/ci/gitlab/defaults/agents/gpu_agent.yml
+  - local: tools/ci/gitlab/defaults/agents/linux_agent.yml
+  - local: tools/ci/gitlab/defaults/agents/windows_agent.yml
+  - local: tools/ci/gitlab/defaults/vault/linux_vault.yml
+  - local: tools/ci/gitlab/defaults/vault/windows_vault.yml
+  # Vault definitions
+  - project: 'omniverse/sectools/vault'
+    file: '/templates/v3/linux/packman_s3.gitlab-ci.yml'
+  - project: 'omniverse/sectools/vault'
+    file: 'templates/v3/linux/codesign.gitlab-ci.yml'
+  - project: 'omniverse/sectools/vault'
+    file: 'templates/v3/windows/packman_s3.gitlab-ci.yml'
+  - project: 'omniverse/sectools/vault'
+    file: 'templates/v3/windows/codesign.gitlab-ci.yml'
@@ -0,0 +1,27 @@
+# Full changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+### Changed
+
+### Fixed
+
+### Removed
+
+## [2024.4.0]
+
+### Added
+- First release
+
+### Changed
+
+### Fixed
+
+### Removed
@@ -0,0 +1 @@
+2024.0.0
@@ -9,6 +9,38 @@ logging = "warn"
 name = "ComfyUI-RTX_Remix"


+########################################################################################################################
+# Package publishing to packman
+########################################################################################################################
+
+[repo_publish]
+enabled = true
+
+packages = ["*"]
+
+########################################################################################################################
+# Packaging
+########################################################################################################################
+
+[repo_package]
+
+[repo_package.packages.main_package]
+
+archive_name = "${conf:repo.name}"
+archive_format = "zip"
+
+windows_max_path_length = 240
+files = [
+    ["**"],
+]
+files_exclude = [
+    ["tools/**"],
+    ["_*/**"],
+    ["repo*"],
+    [".*"],
+]
+
+
 # python is formatted with black+isort:
 [repo_format.python]
 files.include = ["nodes/*.py"]
@@ -2,7 +2,7 @@
 # Tool name
 command = "class_docs"
 # [python module name or path]:[function name]
-entry_point = "${root}/tools/repoman/class_docs.py:setup_repo_tool"
+entry_point = "${root}/tools/utils/class_docs.py:setup_repo_tool"
 # Python install to use
 python_install = "${root}/../../../python_embeded/python.exe"
 # Root module to import
@@ -11,4 +11,18 @@ module = "${root}/nodes/__init__.py"
 # README to modify
 file_path = "${root}/README.md"
 # Header of section where change is required
-section_header = "## Nodes"
+section_header = "## Nodes"
+
+
+[repo_get_release_changelog]
+# Tool name
+command = "get_release_changelog"
+# [python module name or path]:[function name]
+entry_point = "${root}/tools/utils/repo_get_release_changelog.py:setup_repo_tool"
+
+# Name of required file
+file_name = "CHANGELOG.md"
+# Header of section where change is required
+section_header = "[Unreleased]"
+# Pattern delineating sections
+section_pattern = "\n## "
@@ -0,0 +1,37 @@
+build-package-windows:
+  extends:
+    - .defaults
+    - .windows_agent
+  stage: build_package
+  script:
+    - .\repo.bat package --mode main_package --platform windows-x86_64
+  # Artifacts required to make the publishing step work
+  artifacts:
+    when: on_success
+    paths:
+      - _build/packages
+  rules: ## Automatically run publish job only on merge commit to main
+    - if: $CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: always
+    - if: $CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "push"
+      when: always
+    - when: never
+
+build-package-linux:
+  extends:
+    - .defaults
+    - .linux_agent
+  stage: build_package
+  script:
+    - ./repo.sh package --mode main_package --platform linux-x86_64
+  # Artifacts required to make the publishing step work
+  artifacts:
+    when: on_success
+    paths:
+      - _build/packages
+  rules: ## Automatically run publish job only on merge commit to main
+    - if: $CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: always
+    - if: $CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "push"
+      when: always
+    - when: never
@@ -0,0 +1,3 @@
+.gpu_agent:
+  tags:  &gpu_agent_tags
+    - quadro
@@ -0,0 +1,3 @@
+.linux_agent:
+  tags: &linux_agent_tags
+    - linux
@@ -0,0 +1,3 @@
+.windows_agent:
+  tags: &windows_agent_tags
+    - windows
@@ -0,0 +1,7 @@
+.defaults:
+  interruptible: true
+  allow_failure: false
+  variables:
+    GIT_STRATEGY: fetch
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_COMMIT_REF_NAME != "main"
@@ -0,0 +1,8 @@
+.gitlab_release:
+  interruptible: true
+  allow_failure: false
+  stage: gitlab_release
+  rules:
+    - if: $CI_COMMIT_TAG
+      when: always  # Run this job when a tag is created manually
+
@@ -0,0 +1,13 @@
+.linux_vault:
+  extends:
+    - .osec:vault:v3:prod_token_job
+  before_script:
+    - set -eu
+    - |
+      mkdir tempbin
+      curl -fsSL $VAULT_LINUX_URL \
+      | bsdtar -xvf- -C tempbin
+      chmod +x tempbin/vault
+      export PATH="$PWD/tempbin:$PATH"
+    - !reference [ .osec:vault:v3:linux, codesign:perform_vault_requests ]
+    - !reference [ .osec:vault:v3:linux, packman_s3:perform_vault_requests ]
@@ -0,0 +1,6 @@
+.windows_vault:
+  extends:
+    - .osec:vault:v3:prod_token_job
+  before_script:
+    - !reference [ .osec:vault:v3:windows:powershell, packman_s3:perform_vault_requests ]
+    - !reference [ .osec:vault:v3:windows:powershell, codesign:perform_vault_requests ]
@@ -0,0 +1,25 @@
+build-package-gitlab-windows:
+  extends:
+    - build-package-windows
+    - .gitlab_release
+  script:
+    - !reference [build-package-windows, script]
+    - echo (Get-ChildItem  -Filter "_build/packages/*.zip" -Force | ForEach-Object {"ARTEFACT_WIN_ZIP_NAME=$_"}) > gitlab_release_win.env
+  # Artifacts required to make the publishing step work
+  artifacts:
+    reports:
+      # To ensure we've access to this file in the next stage
+      dotenv: gitlab_release_win.env
+
+build-package-gitlab-linux:
+  extends:
+    - build-package-linux
+    - .gitlab_release
+  script:
+    - !reference [build-package-linux, script]
+    - find _build/packages -maxdepth 1 -name "*.zip" -type f -printf "ARTEFACT_LINUX_ZIP_NAME=%f\n" > gitlab_release_linux.env
+  # Artifacts required to make the publishing step work
+  artifacts:
+    reports:
+      # To ensure we've access to this file in the next stage
+      dotenv: gitlab_release_linux.env
@@ -0,0 +1,17 @@
+publish-package-gitlab:
+  extends:
+    - .linux_agent
+    - .linux_vault
+    - .gitlab_release
+  needs:
+    - job: build-package-gitlab-linux
+      artifacts: true
+    - job: build-package-gitlab-windows
+      artifacts: true
+  before_script:
+    - !reference [ .linux_vault, before_script ]
+  script:
+    - fail_wr=0
+    - ./repo.sh publish || fail_wr=1
+    - echo "WinRel=${fail_wr}"
+    - '[ "$fail_wr" -eq 0 ] || exit 1'
@@ -0,0 +1,43 @@
+publish-gitlab-release:
+  extends:
+    - .gitlab_release
+    - .windows_agent
+  image: registry.gitlab.com/gitlab-org/release-cli:latest
+  tags:
+    - release-cli
+  needs:
+    - job: publish-package-gitlab
+      artifacts: true
+    - job: build-package-gitlab-linux  # to get the env
+      artifacts: true
+    - job: build-package-gitlab-windows  # to get the env
+      artifacts: true
+  script:
+    - echo "running release_job for $env:CI_COMMIT_TAG"
+    - echo "grabbing zip file $env:ARTEFACT_WIN_ZIP_NAME and $env:ARTEFACT_LINUX_ZIP_NAME"
+    - .\repo.bat get_release_changelog -s _current_changelog.md -sh "[$env:CI_COMMIT_TAG]"
+    - $env:ENCODED_ARTEFACT_WIN_ZIP_NAME=[URI]::EscapeDataString("$env:ARTEFACT_WIN_ZIP_NAME")
+    - echo "encoded window zip name is $env:ENCODED_ARTEFACT_WIN_ZIP_NAME"
+    - $env:ENCODED_ARTEFACT_LINUX_ZIP_NAME=[URI]::EscapeDataString("$env:ARTEFACT_LINUX_ZIP_NAME")
+    - echo "encoded linux zip name is $env:ENCODED_ARTEFACT_LINUX_ZIP_NAME"
+    - $env:WIN_URL="$env:PACKMAN_CLOUDFRONT_URL/$env:ENCODED_ARTEFACT_WIN_ZIP_NAME"
+    - echo "full asset url for WIN is $env:WIN_URL"
+    - $env:LINUX_URL="$env:PACKMAN_CLOUDFRONT_URL/$env:ENCODED_ARTEFACT_LINUX_ZIP_NAME"
+    - echo "full asset url for LINUX is $env:LINUX_URL"
+    - echo "token is $env:CI_JOB_TOKEN"
+    - echo "token is CI_JOB_TOKEN"
+
+    - $assets_win = @{}
+    - $assets_win.Add("name", "windows-x86_64.release")
+    - $assets_win.Add("url", $env:WIN_URL)
+    - $assets_win = $assets_win | ConvertTo-Json
+    - $assets_win = $assets_win.replace('"','\"')
+
+    - $assets_linux = @{}
+    - $assets_linux.Add("name", "linux-x86_64.release")
+    - $assets_linux.Add("url", $env:LINUX_URL)
+    - $assets_linux = $assets_linux | ConvertTo-Json
+    - $assets_linux = $assets_linux.replace('"','\"')
+
+    # using release feature from the yaml doesn't work with powershell. Need to run the CLI directly
+    - release-cli --private-token $env:GITLAB_RELEASE_TOKEN create --name "$env:CI_COMMIT_TAG" --description "_current_changelog.md" --tag-name "$env:CI_COMMIT_TAG" --ref "$env:CI_COMMIT_SHA" --assets-link=$assets_win --assets-link=$assets_linux
@@ -0,0 +1,20 @@
+$ErrorActionPreference = "Stop"
+
+echo "Running on $([Environment]::MachineName)..."
+
+& {
+    $PSNativeCommandUseErrorActionPreference = $true
+    $ProgressPreference = 'SilentlyContinue'
+    $Url = "$env:VAULT_WINDOWS_URL"
+    $DownloadZipFile = [IO.Path]::Combine((Get-Location).path, $(Split-Path -Path $Url -Leaf))
+    mkdir tempbin
+    Invoke-WebRequest -Uri $Url -OutFile $DownloadZipFile
+    Expand-Archive $DownloadZipFile -DestinationPath tempbin -Force
+    Remove-Item -Path $DownloadZipFile -Force
+    $Env:PATH += ";"
+    $Env:PATH += [IO.Path]::Combine((Get-Location).path, "tempbin")
+}
+
+$ProgressPreference = 'Continue'
+
+if(!$?) { Exit $LASTEXITCODE }
@@ -12,7 +12,7 @@
 :: See the License for the specific language governing permissions and
 :: limitations under the License.

-set PM_PACKMAN_VERSION=7.21.2
+set PM_PACKMAN_VERSION=7.23.1

 :: Specify where packman command is rooted
 set PM_INSTALL_PATH=%~dp0..
@@ -142,7 +142,7 @@ def generate_sha256_for_file(file_path: Union[str, os.PathLike]) -> str:


 def install_common_module(package_path, install_path):
-    COMMON_SHA256 = "e5c8c49148bbaf049cfd27203936486942d5ded0b101d40984659c274b048a38"
+    COMMON_SHA256 = "0a2064434cca0170411c86f23349f9618556dc380d3589a2361db38ffeea9cac"
    package_sha256 = generate_sha256_for_file(package_path)
    if package_sha256 != COMMON_SHA256:
        raise RuntimeError(
@@ -1,6 +1,7 @@
 <config remotes="cloudfront urm">
    <remote2 name="cloudfront">
        <transport actions="download" protocol="http" packageLocation="d4i3qtqj3r0z5.cloudfront.net/${name}@${version}" />
+        <transport actions="upload" protocol="s3" packageLocation="packages-for-cloudfront" />
        <transport actions="list" protocol="http" packageLocation="omnipackages.nvidia.com/api/v1/list/cloudfront" timeout="600" />
    </remote2>
    <remote2 name="urm">
@@ -24,7 +24,7 @@ else
 	PM_CURL_SILENT="-s -S"
 	PM_WGET_QUIET="--quiet"
 fi
-export PM_PACKMAN_VERSION=7.21.2
+export PM_PACKMAN_VERSION=7.23.1

 # This is necessary for newer macOS
 if [ `uname` == 'Darwin' ]; then
@@ -1,4 +1,18 @@
-# Use this file to bootstrap packman into your Python environment (3.7.x). Simply
+# Copyright 2021-2024 NVIDIA CORPORATION
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Use this file to bootstrap packman into your Python environment. Simply
 # add the path by doing sys.insert to where packmanconf.py is located and then execute:
 #
 # >>> import packmanconf
@@ -5,7 +5,10 @@
  <dependency name="repo_format" linkPath="../_repo/deps/repo_format">
    <package name="repo_format" version="2.12.5" />
  </dependency>
-  <dependency name="repo_lint" linkPath="../_repo/deps/repo_lint">
-    <package name="repo_lint" version="1.6.1" />
+  <dependency name="repo_package" linkPath="../_repo/deps/repo_package">
+    <package name="repo_package" version="5.9.12" />
  </dependency>
+<!--  <dependency name="repo_lint" linkPath="../_repo/deps/repo_lint">-->
+<!--    <package name="repo_lint" version="1.6.1" />-->
+<!--  </dependency>-->
 </project>
@@ -0,0 +1,79 @@
+"""
+* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+* SPDX-License-Identifier: Apache-2.0
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* https://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+"""
+import re
+import sys
+
+
+def get_changelog_text(content: str, section_header: str, section_pattern: str):
+    """
+    Get the changelog for a given header
+    """
+    split = [section for section in re.split(section_pattern, content) if section.startswith(section_header)]
+    if not split:
+        print(f"No '{section_header}' section was found")
+        sys.exit(1)
+    return [f"{ln}\n" for ln in split[0].splitlines()[2:]]
+
+
+def setup_repo_tool(parser, _):
+    parser.prog = "get_release_changelog"
+    parser.description = "Get the changelog text for a given release"
+    parser.add_argument(
+        "-s",
+        "--save-current",
+        dest="file_path",
+        required=False,
+        help="Save the current changelog from the version specified in the section_header into a file."
+    )
+    parser.add_argument(
+        "-sh",
+        "--section-header",
+        dest="section_header",
+        required=False,
+        help="Override the section header you want to use",
+    )
+    parser.add_argument(
+        "-sol",
+        "--save-one-line",
+        dest="one_line",
+        action="store_true",
+        required=False,
+        help="To use with -s. If True, it will save the changelog into 1 line",
+    )
+
+    def run_repo_tool(options, config):
+        settings = config["repo_get_release_changelog"]
+        file_name = settings["file_name"]
+        section_pattern = settings["section_pattern"]
+        section_header = options.section_header if options.section_header else settings["section_header"]
+
+        with open(file_name, "r", encoding="utf8") as f:
+            current_text = f.read()
+        current_changelog = get_changelog_text(current_text, section_header, section_pattern)
+        if options.file_path:
+            if options.one_line:
+                with open(options.file_path, "w", encoding="utf8") as f:
+                    f.write(repr("".join(current_changelog)))
+            else:
+                with open(options.file_path, "w", encoding="utf8") as f:
+                    f.writelines(current_changelog)
+            print("Success!")
+        else:
+            print(section_pattern, section_header, sep="", end="\n\n")
+            print("".join(current_changelog))
+
+    return run_repo_tool