Sorcerio-MBM-Music-Visualizer/mbmInterpPromptSequence.py

# MBM's Music Visualizer: Interpolated Prompt Sequence
# Represents and facilitates building of a sequence of prompts with weighted interpolation.

# Imports
import torch

from .mbmPrompt import MbmPrompt

# Classes
class InterpPromptSequence:
    """
    Represents and facilitates building of a sequence of prompts with weighted interpolation.
    """
    # Constructor
    def __init__(self, start: MbmPrompt, end: MbmPrompt, modifiers: torch.Tensor) -> None:
        """
        start: The prompt to start from.
        end: The prompt to end on.
        modifiers: The feature modifiers to use in the weighted interpolation.
        """
        # Calculate initial interpolation
        self.positives = self._weightedInterpolation(
            start.positive,
            end.positive,
            modifiers
        )[0]
        self.negatives = self._weightedInterpolation(
            start.negative,
            end.negative,
            modifiers
        )[0]
        self.positivePools = self._weightedInterpolation(
            start.positivePool,
            end.positivePool,
            modifiers
        )[0]
        self.negativePools = self._weightedInterpolation(
            start.negativePool,
            end.negativePool,
            modifiers
        )[0]

    # Functions
    def addToSequence(self, start: MbmPrompt, end: MbmPrompt, modifiers: torch.Tensor) -> None:
        """
        Add additional interpolated prompts to the sequence.

        start: The prompt to start from.
        end: The prompt to end on.
        modifiers: The feature modifiers to use in the weighted interpolation.
        """
        # Add each data type to the sequence
        self.positives = self._addToSequence(self.positives, start.positive, end.positive, modifiers)
        self.negatives = self._addToSequence(self.negatives, start.negative, end.negative, modifiers)
        self.positivePools = self._addToSequence(self.positivePools, start.positivePool, end.positivePool, modifiers)
        self.negativePools = self._addToSequence(self.negativePools, start.negativePool, end.negativePool, modifiers)

    def _addToSequence(self, container: torch.Tensor, start: torch.Tensor, stop: torch.Tensor, modifiers: torch.Tensor) -> torch.Tensor:
        """
        Add an additional interpolated tensor to the sequence.

        container: The tensor to add the interpolation to.
        start: The tensor to start from.
        stop: The tensor to end on.
        modifiers: The feature modifiers to use in the weighted interpolation.

        Returns the modified `container` tensor.
        """
        # Calculate the interpolation
        interp, shapeChanged = self._weightedInterpolation(
            start,
            stop,
            modifiers,
            tokenCount=container.size(1)
        )
        interp = interp[1:]

        # Check if shape updates are needed
        if shapeChanged:
            container = self.addPromptTokens(container, interp.shape)

        # Create modified container with the interpolation
        return torch.vstack((container, interp))

    def trimToLength(self, length: int) -> None:
        """
        Trims the sequence to the provided length.

        length: The length to trim the sequence to.
        """
        self.positives = self.positives[:length]
        self.negatives = self.negatives[:length]
        self.positivePools = self.positivePools[:length]
        self.negativePools = self.negativePools[:length]

    def asPromptSequence(self) -> list[MbmPrompt]:
        """
        Returns the sequence as a list of MbmPrompt objects.
        """
        return [
            MbmPrompt(
                self.positives[i],
                self.negatives[i],
                positivePool=self.positivePools[i],
                negativePool=self.negativePools[i]
            )
            for i in range(len(self.positives))
        ]

    def _weightedInterpolation(self, start: torch.Tensor, stop: torch.Tensor, weights: torch.Tensor, tokenCount: int = -1) -> tuple[torch.Tensor, bool]:
        """
        Interpolates between `start` and `stop` based on the given `weights` for each step in the interpolation.

        start: A Tensor of the same shape as `stop`.
        stop: A Tensor of the same shape as `start`.
        weights: A Tensor of weights to to use in each jump of the interpolation. This defines the number of gaps, not the total number of output Tensor elements.
        tokenCount: The number of tokens in the prompt tensor. If the number of tokens in the prompt tensor is less than this value, zero-padding will be added to match the length.

        Returns a tuple of:
        * A Tensor of shape `[(length of weights + 1), *start.shape]` where each step is an interpolation between `start` and `stop`. Includes the `start` or `stop` tensors in the output.
        * A boolean indicating if the shape of the output Tensor has changed.
        """
        # Make sure weights are floats
        weights = weights.float()

        # Normalize weights
        weights = weights / weights.sum()

        # Calculate the cumulative sum of the weights
        cumWeight = weights.cumsum(dim=0)

        # Reshape the cumulative sum to allow for broadcasting
        for _ in range(start.ndim): # [-1, *([1]*start.ndim)]
            cumWeight = cumWeight.unsqueeze(-1)

        # Resample the shorter tensor if needed
        shapeChanged = False
        maxLen = max(start.size(0), stop.size(0), tokenCount)

        if start.size(0) < maxLen:
            shapeChanged = True
            start = torch.vstack([start, torch.zeros(maxLen - start.size(0), start.size(1))])

        if stop.size(0) < maxLen:
            shapeChanged = True
            stop = torch.vstack([stop, torch.zeros(maxLen - stop.size(0), stop.size(1))])

        # Interpolate with weights and add start
        return (
            torch.vstack([start.unsqueeze(0), (start[None] + cumWeight * (stop - start)[None])]),
            shapeChanged
        )

    def addPromptTokens(self, tensor: torch.Tensor, shape: tuple[int, ...]) -> torch.Tensor:
        """
        Adds zero-padding to the given prompt tensor to match the provided `shape` along the dimension that represents the number of tokens in the prompt tensor.

        tensor: The prompt tensor to add zero-padding to.
        shape: The shape to match the prompt tensor to.

        Returns the modified tensor or the original tensor if no padding is needed.
        """
        # Calculate the size of the zero-padding needed
        paddingSize = shape[1] - tensor.shape[1]

        # Check if padding is needed
        if paddingSize <= 0:
            return tensor

        # Concatenate the original tensor and the zero-padding along dimension 1
        return torch.cat(
            [
                tensor,
                torch.zeros(tensor.shape[0], paddingSize, tensor.shape[2], dtype=tensor.dtype, device=tensor.device)
            ],
            dim=1
        )