From 32328d4e72963f56e9cc8b2af595359692b031e1 Mon Sep 17 00:00:00 2001 From: Damien Bataille Date: Fri, 19 Jul 2024 16:43:26 -0700 Subject: [PATCH] Add versioning + CI to publish in Packman --- .gitlab-ci.yml | 35 ++++++++ CHANGELOG.md | 27 +++++++ VERSION.md | 1 + repo.toml | 32 ++++++++ repo_tools.toml | 18 ++++- tools/ci/gitlab/build/build_package.yml | 37 +++++++++ tools/ci/gitlab/defaults/agents/gpu_agent.yml | 3 + .../ci/gitlab/defaults/agents/linux_agent.yml | 3 + .../gitlab/defaults/agents/windows_agent.yml | 3 + tools/ci/gitlab/defaults/defaults.yml | 7 ++ tools/ci/gitlab/defaults/gitlab_release.yml | 8 ++ .../ci/gitlab/defaults/vault/linux_vault.yml | 13 +++ .../gitlab/defaults/vault/windows_vault.yml | 6 ++ .../ci/gitlab/gitlab/build_gitlab_package.yml | 25 ++++++ tools/ci/gitlab/gitlab/publish_package.yml | 17 ++++ tools/ci/gitlab/gitlab/publish_release.yml | 43 ++++++++++ tools/ci/vault/download_vault.ps1 | 20 +++++ tools/packman/bootstrap/configure.bat | 2 +- tools/packman/bootstrap/install_package.py | 2 +- tools/packman/config.packman.xml | 1 + tools/packman/packman | 2 +- tools/packman/packmanconf.py | 16 +++- tools/repo-deps.packman.xml | 7 +- tools/{repoman => utils}/class_docs.py | 0 tools/utils/repo_get_release_changelog.py | 79 +++++++++++++++++++ 25 files changed, 399 insertions(+), 8 deletions(-) create mode 100644 .gitlab-ci.yml create mode 100644 CHANGELOG.md create mode 100644 VERSION.md create mode 100644 tools/ci/gitlab/build/build_package.yml create mode 100644 tools/ci/gitlab/defaults/agents/gpu_agent.yml create mode 100644 tools/ci/gitlab/defaults/agents/linux_agent.yml create mode 100644 tools/ci/gitlab/defaults/agents/windows_agent.yml create mode 100644 tools/ci/gitlab/defaults/defaults.yml create mode 100644 tools/ci/gitlab/defaults/gitlab_release.yml create mode 100644 tools/ci/gitlab/defaults/vault/linux_vault.yml create mode 100644 tools/ci/gitlab/defaults/vault/windows_vault.yml create mode 100644 tools/ci/gitlab/gitlab/build_gitlab_package.yml create mode 100644 tools/ci/gitlab/gitlab/publish_package.yml create mode 100644 tools/ci/gitlab/gitlab/publish_release.yml create mode 100644 tools/ci/vault/download_vault.ps1 rename tools/{repoman => utils}/class_docs.py (100%) create mode 100644 tools/utils/repo_get_release_changelog.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..f5daf5c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,35 @@ +variables: + AWS_ACCESS_KEY_ID: $AWS_OMNIVERSEDOCS_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_OMNIVERSEDOCS_SECRET_ACCESS_KEY + DXVK_REMIX_NVIDIA_MIN_DRIVER: 0 + +stages: + - build_package + - publish_extensions + - publish_package + - publish_package_scheduled + - gitlab_release + +include: + # Job definitions + - local: tools/ci/gitlab/build/build_package.yml + - local: tools/ci/gitlab/gitlab/build_gitlab_package.yml + - local: tools/ci/gitlab/gitlab/publish_package.yml + - local: tools/ci/gitlab/gitlab/publish_release.yml + # Base definitions shared between jobs + - local: tools/ci/gitlab/defaults/defaults.yml + - local: tools/ci/gitlab/defaults/gitlab_release.yml + - local: tools/ci/gitlab/defaults/agents/gpu_agent.yml + - local: tools/ci/gitlab/defaults/agents/linux_agent.yml + - local: tools/ci/gitlab/defaults/agents/windows_agent.yml + - local: tools/ci/gitlab/defaults/vault/linux_vault.yml + - local: tools/ci/gitlab/defaults/vault/windows_vault.yml + # Vault definitions + - project: 'omniverse/sectools/vault' + file: '/templates/v3/linux/packman_s3.gitlab-ci.yml' + - project: 'omniverse/sectools/vault' + file: 'templates/v3/linux/codesign.gitlab-ci.yml' + - project: 'omniverse/sectools/vault' + file: 'templates/v3/windows/packman_s3.gitlab-ci.yml' + - project: 'omniverse/sectools/vault' + file: 'templates/v3/windows/codesign.gitlab-ci.yml' diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..56b4ca1 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,27 @@ +# Full changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added + +### Changed + +### Fixed + +### Removed + +## [2024.4.0] + +### Added +- First release + +### Changed + +### Fixed + +### Removed diff --git a/VERSION.md b/VERSION.md new file mode 100644 index 0000000..fec75bd --- /dev/null +++ b/VERSION.md @@ -0,0 +1 @@ +2024.0.0 diff --git a/repo.toml b/repo.toml index ccb3b18..8cab870 100644 --- a/repo.toml +++ b/repo.toml @@ -9,6 +9,38 @@ logging = "warn" name = "ComfyUI-RTX_Remix" +######################################################################################################################## +# Package publishing to packman +######################################################################################################################## + +[repo_publish] +enabled = true + +packages = ["*"] + +######################################################################################################################## +# Packaging +######################################################################################################################## + +[repo_package] + +[repo_package.packages.main_package] + +archive_name = "${conf:repo.name}" +archive_format = "zip" + +windows_max_path_length = 240 +files = [ + ["**"], +] +files_exclude = [ + ["tools/**"], + ["_*/**"], + ["repo*"], + [".*"], +] + + # python is formatted with black+isort: [repo_format.python] files.include = ["nodes/*.py"] diff --git a/repo_tools.toml b/repo_tools.toml index 80170e2..cab1525 100644 --- a/repo_tools.toml +++ b/repo_tools.toml @@ -2,7 +2,7 @@ # Tool name command = "class_docs" # [python module name or path]:[function name] -entry_point = "${root}/tools/repoman/class_docs.py:setup_repo_tool" +entry_point = "${root}/tools/utils/class_docs.py:setup_repo_tool" # Python install to use python_install = "${root}/../../../python_embeded/python.exe" # Root module to import @@ -11,4 +11,18 @@ module = "${root}/nodes/__init__.py" # README to modify file_path = "${root}/README.md" # Header of section where change is required -section_header = "## Nodes" \ No newline at end of file +section_header = "## Nodes" + + +[repo_get_release_changelog] +# Tool name +command = "get_release_changelog" +# [python module name or path]:[function name] +entry_point = "${root}/tools/utils/repo_get_release_changelog.py:setup_repo_tool" + +# Name of required file +file_name = "CHANGELOG.md" +# Header of section where change is required +section_header = "[Unreleased]" +# Pattern delineating sections +section_pattern = "\n## " diff --git a/tools/ci/gitlab/build/build_package.yml b/tools/ci/gitlab/build/build_package.yml new file mode 100644 index 0000000..2634eb8 --- /dev/null +++ b/tools/ci/gitlab/build/build_package.yml @@ -0,0 +1,37 @@ +build-package-windows: + extends: + - .defaults + - .windows_agent + stage: build_package + script: + - .\repo.bat package --mode main_package --platform windows-x86_64 + # Artifacts required to make the publishing step work + artifacts: + when: on_success + paths: + - _build/packages + rules: ## Automatically run publish job only on merge commit to main + - if: $CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "merge_request_event" + when: always + - if: $CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "push" + when: always + - when: never + +build-package-linux: + extends: + - .defaults + - .linux_agent + stage: build_package + script: + - ./repo.sh package --mode main_package --platform linux-x86_64 + # Artifacts required to make the publishing step work + artifacts: + when: on_success + paths: + - _build/packages + rules: ## Automatically run publish job only on merge commit to main + - if: $CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "merge_request_event" + when: always + - if: $CI_COMMIT_REF_NAME == "main" && $CI_PIPELINE_SOURCE == "push" + when: always + - when: never diff --git a/tools/ci/gitlab/defaults/agents/gpu_agent.yml b/tools/ci/gitlab/defaults/agents/gpu_agent.yml new file mode 100644 index 0000000..42a742a --- /dev/null +++ b/tools/ci/gitlab/defaults/agents/gpu_agent.yml @@ -0,0 +1,3 @@ +.gpu_agent: + tags: &gpu_agent_tags + - quadro diff --git a/tools/ci/gitlab/defaults/agents/linux_agent.yml b/tools/ci/gitlab/defaults/agents/linux_agent.yml new file mode 100644 index 0000000..96e61de --- /dev/null +++ b/tools/ci/gitlab/defaults/agents/linux_agent.yml @@ -0,0 +1,3 @@ +.linux_agent: + tags: &linux_agent_tags + - linux diff --git a/tools/ci/gitlab/defaults/agents/windows_agent.yml b/tools/ci/gitlab/defaults/agents/windows_agent.yml new file mode 100644 index 0000000..ad1637c --- /dev/null +++ b/tools/ci/gitlab/defaults/agents/windows_agent.yml @@ -0,0 +1,3 @@ +.windows_agent: + tags: &windows_agent_tags + - windows diff --git a/tools/ci/gitlab/defaults/defaults.yml b/tools/ci/gitlab/defaults/defaults.yml new file mode 100644 index 0000000..6c45381 --- /dev/null +++ b/tools/ci/gitlab/defaults/defaults.yml @@ -0,0 +1,7 @@ +.defaults: + interruptible: true + allow_failure: false + variables: + GIT_STRATEGY: fetch + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_COMMIT_REF_NAME != "main" diff --git a/tools/ci/gitlab/defaults/gitlab_release.yml b/tools/ci/gitlab/defaults/gitlab_release.yml new file mode 100644 index 0000000..b32c8c1 --- /dev/null +++ b/tools/ci/gitlab/defaults/gitlab_release.yml @@ -0,0 +1,8 @@ +.gitlab_release: + interruptible: true + allow_failure: false + stage: gitlab_release + rules: + - if: $CI_COMMIT_TAG + when: always # Run this job when a tag is created manually + diff --git a/tools/ci/gitlab/defaults/vault/linux_vault.yml b/tools/ci/gitlab/defaults/vault/linux_vault.yml new file mode 100644 index 0000000..637424d --- /dev/null +++ b/tools/ci/gitlab/defaults/vault/linux_vault.yml @@ -0,0 +1,13 @@ +.linux_vault: + extends: + - .osec:vault:v3:prod_token_job + before_script: + - set -eu + - | + mkdir tempbin + curl -fsSL $VAULT_LINUX_URL \ + | bsdtar -xvf- -C tempbin + chmod +x tempbin/vault + export PATH="$PWD/tempbin:$PATH" + - !reference [ .osec:vault:v3:linux, codesign:perform_vault_requests ] + - !reference [ .osec:vault:v3:linux, packman_s3:perform_vault_requests ] diff --git a/tools/ci/gitlab/defaults/vault/windows_vault.yml b/tools/ci/gitlab/defaults/vault/windows_vault.yml new file mode 100644 index 0000000..94c9a76 --- /dev/null +++ b/tools/ci/gitlab/defaults/vault/windows_vault.yml @@ -0,0 +1,6 @@ +.windows_vault: + extends: + - .osec:vault:v3:prod_token_job + before_script: + - !reference [ .osec:vault:v3:windows:powershell, packman_s3:perform_vault_requests ] + - !reference [ .osec:vault:v3:windows:powershell, codesign:perform_vault_requests ] diff --git a/tools/ci/gitlab/gitlab/build_gitlab_package.yml b/tools/ci/gitlab/gitlab/build_gitlab_package.yml new file mode 100644 index 0000000..1a78771 --- /dev/null +++ b/tools/ci/gitlab/gitlab/build_gitlab_package.yml @@ -0,0 +1,25 @@ +build-package-gitlab-windows: + extends: + - build-package-windows + - .gitlab_release + script: + - !reference [build-package-windows, script] + - echo (Get-ChildItem -Filter "_build/packages/*.zip" -Force | ForEach-Object {"ARTEFACT_WIN_ZIP_NAME=$_"}) > gitlab_release_win.env + # Artifacts required to make the publishing step work + artifacts: + reports: + # To ensure we've access to this file in the next stage + dotenv: gitlab_release_win.env + +build-package-gitlab-linux: + extends: + - build-package-linux + - .gitlab_release + script: + - !reference [build-package-linux, script] + - find _build/packages -maxdepth 1 -name "*.zip" -type f -printf "ARTEFACT_LINUX_ZIP_NAME=%f\n" > gitlab_release_linux.env + # Artifacts required to make the publishing step work + artifacts: + reports: + # To ensure we've access to this file in the next stage + dotenv: gitlab_release_linux.env diff --git a/tools/ci/gitlab/gitlab/publish_package.yml b/tools/ci/gitlab/gitlab/publish_package.yml new file mode 100644 index 0000000..0f16459 --- /dev/null +++ b/tools/ci/gitlab/gitlab/publish_package.yml @@ -0,0 +1,17 @@ +publish-package-gitlab: + extends: + - .linux_agent + - .linux_vault + - .gitlab_release + needs: + - job: build-package-gitlab-linux + artifacts: true + - job: build-package-gitlab-windows + artifacts: true + before_script: + - !reference [ .linux_vault, before_script ] + script: + - fail_wr=0 + - ./repo.sh publish || fail_wr=1 + - echo "WinRel=${fail_wr}" + - '[ "$fail_wr" -eq 0 ] || exit 1' diff --git a/tools/ci/gitlab/gitlab/publish_release.yml b/tools/ci/gitlab/gitlab/publish_release.yml new file mode 100644 index 0000000..d442146 --- /dev/null +++ b/tools/ci/gitlab/gitlab/publish_release.yml @@ -0,0 +1,43 @@ +publish-gitlab-release: + extends: + - .gitlab_release + - .windows_agent + image: registry.gitlab.com/gitlab-org/release-cli:latest + tags: + - release-cli + needs: + - job: publish-package-gitlab + artifacts: true + - job: build-package-gitlab-linux # to get the env + artifacts: true + - job: build-package-gitlab-windows # to get the env + artifacts: true + script: + - echo "running release_job for $env:CI_COMMIT_TAG" + - echo "grabbing zip file $env:ARTEFACT_WIN_ZIP_NAME and $env:ARTEFACT_LINUX_ZIP_NAME" + - .\repo.bat get_release_changelog -s _current_changelog.md -sh "[$env:CI_COMMIT_TAG]" + - $env:ENCODED_ARTEFACT_WIN_ZIP_NAME=[URI]::EscapeDataString("$env:ARTEFACT_WIN_ZIP_NAME") + - echo "encoded window zip name is $env:ENCODED_ARTEFACT_WIN_ZIP_NAME" + - $env:ENCODED_ARTEFACT_LINUX_ZIP_NAME=[URI]::EscapeDataString("$env:ARTEFACT_LINUX_ZIP_NAME") + - echo "encoded linux zip name is $env:ENCODED_ARTEFACT_LINUX_ZIP_NAME" + - $env:WIN_URL="$env:PACKMAN_CLOUDFRONT_URL/$env:ENCODED_ARTEFACT_WIN_ZIP_NAME" + - echo "full asset url for WIN is $env:WIN_URL" + - $env:LINUX_URL="$env:PACKMAN_CLOUDFRONT_URL/$env:ENCODED_ARTEFACT_LINUX_ZIP_NAME" + - echo "full asset url for LINUX is $env:LINUX_URL" + - echo "token is $env:CI_JOB_TOKEN" + - echo "token is CI_JOB_TOKEN" + + - $assets_win = @{} + - $assets_win.Add("name", "windows-x86_64.release") + - $assets_win.Add("url", $env:WIN_URL) + - $assets_win = $assets_win | ConvertTo-Json + - $assets_win = $assets_win.replace('"','\"') + + - $assets_linux = @{} + - $assets_linux.Add("name", "linux-x86_64.release") + - $assets_linux.Add("url", $env:LINUX_URL) + - $assets_linux = $assets_linux | ConvertTo-Json + - $assets_linux = $assets_linux.replace('"','\"') + + # using release feature from the yaml doesn't work with powershell. Need to run the CLI directly + - release-cli --private-token $env:GITLAB_RELEASE_TOKEN create --name "$env:CI_COMMIT_TAG" --description "_current_changelog.md" --tag-name "$env:CI_COMMIT_TAG" --ref "$env:CI_COMMIT_SHA" --assets-link=$assets_win --assets-link=$assets_linux diff --git a/tools/ci/vault/download_vault.ps1 b/tools/ci/vault/download_vault.ps1 new file mode 100644 index 0000000..769fa29 --- /dev/null +++ b/tools/ci/vault/download_vault.ps1 @@ -0,0 +1,20 @@ +$ErrorActionPreference = "Stop" + +echo "Running on $([Environment]::MachineName)..." + +& { + $PSNativeCommandUseErrorActionPreference = $true + $ProgressPreference = 'SilentlyContinue' + $Url = "$env:VAULT_WINDOWS_URL" + $DownloadZipFile = [IO.Path]::Combine((Get-Location).path, $(Split-Path -Path $Url -Leaf)) + mkdir tempbin + Invoke-WebRequest -Uri $Url -OutFile $DownloadZipFile + Expand-Archive $DownloadZipFile -DestinationPath tempbin -Force + Remove-Item -Path $DownloadZipFile -Force + $Env:PATH += ";" + $Env:PATH += [IO.Path]::Combine((Get-Location).path, "tempbin") +} + +$ProgressPreference = 'Continue' + +if(!$?) { Exit $LASTEXITCODE } diff --git a/tools/packman/bootstrap/configure.bat b/tools/packman/bootstrap/configure.bat index c8ea829..33fb441 100644 --- a/tools/packman/bootstrap/configure.bat +++ b/tools/packman/bootstrap/configure.bat @@ -12,7 +12,7 @@ :: See the License for the specific language governing permissions and :: limitations under the License. -set PM_PACKMAN_VERSION=7.21.2 +set PM_PACKMAN_VERSION=7.23.1 :: Specify where packman command is rooted set PM_INSTALL_PATH=%~dp0.. diff --git a/tools/packman/bootstrap/install_package.py b/tools/packman/bootstrap/install_package.py index 240dd02..4542c4b 100644 --- a/tools/packman/bootstrap/install_package.py +++ b/tools/packman/bootstrap/install_package.py @@ -142,7 +142,7 @@ def generate_sha256_for_file(file_path: Union[str, os.PathLike]) -> str: def install_common_module(package_path, install_path): - COMMON_SHA256 = "e5c8c49148bbaf049cfd27203936486942d5ded0b101d40984659c274b048a38" + COMMON_SHA256 = "0a2064434cca0170411c86f23349f9618556dc380d3589a2361db38ffeea9cac" package_sha256 = generate_sha256_for_file(package_path) if package_sha256 != COMMON_SHA256: raise RuntimeError( diff --git a/tools/packman/config.packman.xml b/tools/packman/config.packman.xml index e7ba725..c548b6a 100644 --- a/tools/packman/config.packman.xml +++ b/tools/packman/config.packman.xml @@ -1,6 +1,7 @@ + diff --git a/tools/packman/packman b/tools/packman/packman index d2c2af7..b222d30 100644 --- a/tools/packman/packman +++ b/tools/packman/packman @@ -24,7 +24,7 @@ else PM_CURL_SILENT="-s -S" PM_WGET_QUIET="--quiet" fi -export PM_PACKMAN_VERSION=7.21.2 +export PM_PACKMAN_VERSION=7.23.1 # This is necessary for newer macOS if [ `uname` == 'Darwin' ]; then diff --git a/tools/packman/packmanconf.py b/tools/packman/packmanconf.py index 6afae51..db220d2 100644 --- a/tools/packman/packmanconf.py +++ b/tools/packman/packmanconf.py @@ -1,4 +1,18 @@ -# Use this file to bootstrap packman into your Python environment (3.7.x). Simply +# Copyright 2021-2024 NVIDIA CORPORATION + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Use this file to bootstrap packman into your Python environment. Simply # add the path by doing sys.insert to where packmanconf.py is located and then execute: # # >>> import packmanconf diff --git a/tools/repo-deps.packman.xml b/tools/repo-deps.packman.xml index 665c8e3..94b3fa2 100644 --- a/tools/repo-deps.packman.xml +++ b/tools/repo-deps.packman.xml @@ -5,7 +5,10 @@ - - + + + + + diff --git a/tools/repoman/class_docs.py b/tools/utils/class_docs.py similarity index 100% rename from tools/repoman/class_docs.py rename to tools/utils/class_docs.py diff --git a/tools/utils/repo_get_release_changelog.py b/tools/utils/repo_get_release_changelog.py new file mode 100644 index 0000000..9cd0b95 --- /dev/null +++ b/tools/utils/repo_get_release_changelog.py @@ -0,0 +1,79 @@ +""" +* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +* SPDX-License-Identifier: Apache-2.0 +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* https://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +""" +import re +import sys + + +def get_changelog_text(content: str, section_header: str, section_pattern: str): + """ + Get the changelog for a given header + """ + split = [section for section in re.split(section_pattern, content) if section.startswith(section_header)] + if not split: + print(f"No '{section_header}' section was found") + sys.exit(1) + return [f"{ln}\n" for ln in split[0].splitlines()[2:]] + + +def setup_repo_tool(parser, _): + parser.prog = "get_release_changelog" + parser.description = "Get the changelog text for a given release" + parser.add_argument( + "-s", + "--save-current", + dest="file_path", + required=False, + help="Save the current changelog from the version specified in the section_header into a file." + ) + parser.add_argument( + "-sh", + "--section-header", + dest="section_header", + required=False, + help="Override the section header you want to use", + ) + parser.add_argument( + "-sol", + "--save-one-line", + dest="one_line", + action="store_true", + required=False, + help="To use with -s. If True, it will save the changelog into 1 line", + ) + + def run_repo_tool(options, config): + settings = config["repo_get_release_changelog"] + file_name = settings["file_name"] + section_pattern = settings["section_pattern"] + section_header = options.section_header if options.section_header else settings["section_header"] + + with open(file_name, "r", encoding="utf8") as f: + current_text = f.read() + current_changelog = get_changelog_text(current_text, section_header, section_pattern) + if options.file_path: + if options.one_line: + with open(options.file_path, "w", encoding="utf8") as f: + f.write(repr("".join(current_changelog))) + else: + with open(options.file_path, "w", encoding="utf8") as f: + f.writelines(current_changelog) + print("Success!") + else: + print(section_pattern, section_header, sep="", end="\n\n") + print("".join(current_changelog)) + + return run_repo_tool