diff --git a/CHANGELOG.md b/CHANGELOG.md index dc310b22..171f16eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## Unreleased +### The Model Manager no longer reports an install that did not happen + +After the server refused a model install request, the Model Manager kept +waiting, and the end of the next queue, such as a node pack install, was taken +for the end of that install: it asked to click Refresh and raised the notice +that all model downloads were completed, although no model had been installed. +The end of a queue that carries no model result now returns the dialog to idle +without a message when no task is left, and leaves a pending install and its +Stop button alone while tasks are left (#3437). + ### A snapshot that cannot be restored or removed says why When the server refused to restore or remove a snapshot, the Snapshot Manager diff --git a/js/model-manager.js b/js/model-manager.js index 74715112..d3788010 100644 --- a/js/model-manager.js +++ b/js/model-manager.js @@ -501,7 +501,8 @@ export class ModelManager { } } - this.install_context = {btn: btn, targets: target_items}; + const context = {btn: btn, targets: target_items}; + this.install_context = context; if(errorMsg) { this.showError(errorMsg); @@ -515,7 +516,10 @@ export class ModelManager { } else { await api.fetchApi('/manager/queue/start', { method: 'POST' }); - this.showStop(); + // The queue can end before this request is answered. + if(this.install_context === context) { + this.showStop(); + } showTerminal(); } } @@ -535,24 +539,36 @@ export class ModelManager { self.grid.updateRow(item); } else if(event.detail.status == 'done') { - self.hideStop(); self.onQueueCompleted(event.detail); } } async onQueueCompleted(info) { let result = info.model_result; - - if(result.length == 0) { - return; - } - let self = ModelManager.instance; if(!self.install_context) { return; } + // model_result is a dict (model hash -> outcome). A queue that ends + // without one, such as a node pack install, is not this install's + // while tasks are left. With none left, this install is over without + // a result, as after a refused request or a queue that was reset. + const ended_without_result = !result || Object.keys(result).length === 0; + + if(ended_without_result) { + // Another install can start while the server is asked. + const context = self.install_context; + const stats = await (await api.fetchApi('/manager/queue/status')).json(); + + if(stats.total_count > stats.done_count || self.install_context !== context) { + return; + } + } + + self.hideStop(); + let btn = self.install_context.btn; self.hideLoading(); @@ -573,11 +589,16 @@ export class ModelManager { self.grid.updateCell(item, "installed"); } + if(ended_without_result) { + self.install_context = undefined; + return; + } + if (errorMsg) { self.showError(errorMsg); show_message("Installation Error:\n"+errorMsg); } else { - self.showStatus(`Install ${result.length} models successfully`); + self.showStatus(`Install ${Object.keys(result).length} models successfully`); } self.showRefresh(); diff --git a/tests/test_message_sink_provenance.py b/tests/test_message_sink_provenance.py index 5862dda0..b0158308 100644 --- a/tests/test_message_sink_provenance.py +++ b/tests/test_message_sink_provenance.py @@ -136,7 +136,8 @@ REGISTRY = { "item.name comes from /externalmodel/getlist, where populate_markdown -> " "sanitize_tag has already escaped it. WI-112 added a caller escape here " "and it rendered '<Flux>' to the user; removing it is the fix"), - ("model-manager.js", "showStatus", "`Install ${result.length} models successfully`);"): + ("model-manager.js", "showStatus", + "`Install ${Object.keys(result).length} models successfully`);"): (NO_DATA, "a count of queue results"), ("model-manager.js", "showStatus", "`Loading external model list ...`);"): (NO_DATA, "hardcoded"), diff --git a/tests/test_model_queue_completion.py b/tests/test_model_queue_completion.py new file mode 100644 index 00000000..3bd9ed00 --- /dev/null +++ b/tests/test_model_queue_completion.py @@ -0,0 +1,196 @@ +"""The Model Manager ends its pending install when its own queue ends. + +The server reports the end of every queue with two dicts, `nodepack_result` and +`model_result`, keyed by the id of each task. A queue that ends with model results ends +the install and reports them. A queue that ends without one is another one's while tasks +are left, and leaves the install pending with its Stop button in place. With no task +left, the install is over without a result, as when a node pack queue ends after a model +install request was refused, and the dialog returns to idle without reporting anything. + +The cases run the shipped `onQueueStatus` and `onQueueCompleted` of model-manager.js in +node, against a stand-in for the dialog that records what the handlers do to it and a +stand-in for `/manager/queue/status`. The last cases run `installModels` as well, with a +queue that is still running when the request that started it is answered, and with one +that has ended before. +""" +import json +import os +from pathlib import Path + +import pytest + +from js_lift import NODE, JsSource, run_node, slice_braced + +JS = JsSource(Path(os.environ.get("MANAGER_JS_DIR") or (Path(__file__).resolve().parent.parent / "js"))) + +pytestmark = pytest.mark.skipif(NODE is None, reason="node is required to execute the lifted production JS") + +OTHER_QUEUE = {"status": "done", "nodepack_result": {"pack": "success"}, "model_result": {}} +REFRESH = "To apply the installed model, please click the 'Refresh' button." + + +DOWNLOADING = {"total_count": 2, "done_count": 0, "in_progress_count": 1, "is_processing": True} +NOTHING_LEFT = {"total_count": 0, "done_count": 0, "in_progress_count": 0, "is_processing": False} +# the server keeps the model results of the queue before until the next one is started +EARLIER_RESULTS_ONLY = {"total_count": 1, "done_count": 1, "in_progress_count": 0, "is_processing": False} + + +def run(events, queue_status=DOWNLOADING, install_started_while_asking=False): + """Queue two models, deliver `events`, and return what the handlers did after each one. + + `queue_status` is what the server answers when it is asked what is left in the queue. + With `install_started_while_asking`, another install becomes the pending one before + that answer arrives.""" + source = JS.text("model-manager.js") + script = """ + %s + const calls = []; + const self = { + install_context: { + btn: {classList: {remove: () => calls.push('button idle')}}, + targets: [{hash: 'model-a'}, {hash: 'model-b'}], + }, + grid: {updateCell: (item) => calls.push('cell ' + item.hash)}, + hideLoading() {}, + hideStop: () => calls.push('stop hidden'), + showRefresh: () => calls.push('refresh shown'), + showStatus: (text) => calls.push('status: ' + text), + showError: (text) => calls.push('error: ' + text), + showMessage: (text) => calls.push('message: ' + text), + %s, + %s, + }; + const ModelManager = {instance: self}; + const api = {fetchApi: async (path) => { + calls.push('asked ' + path); + if(%s) { + self.install_context = { + btn: {classList: {remove: () => calls.push('button of the new install idle')}}, + targets: [{hash: 'model-c'}], + }; + } + return {json: async () => (%s)}; + }}; + const show_message = (text) => calls.push('dialog: ' + text); + const infoToast = () => calls.push('toast'); + + const steps = []; + for(const detail of %s) { + calls.length = 0; + await self.onQueueStatus({detail}); + // onQueueStatus does not wait for the completion, which may ask the server first + await new Promise((resolve) => setTimeout(resolve)); + steps.push({pending: self.install_context !== undefined, calls: [...calls]}); + } + console.log(JSON.stringify(steps)); + """ % ( + JS.lift_declaration("common.js", "export function sanitizeHTML("), + slice_braced(source, "async onQueueStatus(event)"), + slice_braced(source, "async onQueueCompleted(info)"), + json.dumps(install_started_while_asking), + json.dumps(queue_status), + json.dumps(events), + ) + return run_node(script) + + +@pytest.mark.parametrize("event", [ + OTHER_QUEUE, + {"status": "done", "nodepack_result": {"pack": "success"}}, +]) +def test_queue_without_a_model_result_leaves_the_install_pending(event): + assert run([event]) == [{"pending": True, "calls": ["asked /manager/queue/status"]}] + + +@pytest.mark.parametrize("queue_status", [NOTHING_LEFT, EARLIER_RESULTS_ONLY]) +def test_install_that_ends_without_a_result_returns_to_idle(queue_status): + assert run([OTHER_QUEUE], queue_status=queue_status) == [{"pending": False, "calls": [ + "asked /manager/queue/status", "stop hidden", "button idle", "cell model-a", "cell model-b", + ]}] + + +def test_install_started_while_the_server_is_asked_stays_pending(): + assert run([OTHER_QUEUE], queue_status=NOTHING_LEFT, install_started_while_asking=True) == [ + {"pending": True, "calls": ["asked /manager/queue/status"]}, + ] + + +def test_queue_with_the_model_results_ends_the_install(): + own_queue = {"status": "done", "nodepack_result": {}, + "model_result": {"model-a": "success", "model-b": "success"}} + + other, own = run([OTHER_QUEUE, own_queue]) + + assert other == {"pending": True, "calls": ["asked /manager/queue/status"]} + assert own == {"pending": False, "calls": [ + "stop hidden", "button idle", "cell model-a", "cell model-b", + "status: Install 2 models successfully", "refresh shown", "message: " + REFRESH, "toast", + ]} + + +def test_failed_download_is_reported_as_text(): + failure = "Model installation error: https://example.invalid/a.safetensors" + escaped = "Model installation error: <b>https://example.invalid/a.safetensors</b>\n" + + (step,) = run([{"status": "done", "nodepack_result": {}, + "model_result": {"model-a": failure, "model-b": "success"}}]) + + assert step["pending"] is False + assert "error: " + escaped in step["calls"] + assert "dialog: Installation Error:\n" + escaped in step["calls"] + assert not [call for call in step["calls"] if call.startswith("status: ")] + + +def install(queue_ends_before_the_start_is_answered): + """Install one model, and return whether the install is pending and the Stop button shown.""" + source = JS.text("model-manager.js") + script = """ + %s + let stop_shown = false; + const self = { + grid: {scrollRowIntoView() {}, updateCell() {}}, + focusInstall: () => true, + hideLoading() {}, + showStop: () => { stop_shown = true; }, + hideStop: () => { stop_shown = false; }, + showRefresh() {}, showStatus() {}, showError() {}, showMessage() {}, + %s, + %s, + %s, + }; + const ModelManager = {instance: self}; + const api = {fetchApi: async (path) => { + if(path == '/manager/queue/status') { + return {json: async () => ({is_processing: false, total_count: 0, done_count: 0})}; + } + if(path == '/manager/queue/start' && %s) { + // a download that fails at once: the queue ends while the request is on its way + await self.onQueueStatus({detail: { + status: 'done', nodepack_result: {}, model_result: {'model-a': 'download failed'}, + }}); + } + return {status: 200}; + }}; + const show_message = () => {}; + const showTerminal = () => {}; + const infoToast = () => {}; + + const button = {classList: {add() {}, remove() {}}}; + await self.installModels([{hash: 'model-a', name: 'a', originalData: {}}], button); + console.log(JSON.stringify({pending: self.install_context !== undefined, stop_shown})); + """ % ( + JS.lift_declaration("common.js", "export function sanitizeHTML("), + slice_braced(source, "async installModels(list, btn)"), + slice_braced(source, "async onQueueStatus(event)"), + slice_braced(source, "async onQueueCompleted(info)"), + json.dumps(queue_ends_before_the_start_is_answered), + ) + return run_node(script) + + +def test_started_install_is_pending_with_its_stop_button(): + assert install(queue_ends_before_the_start_is_answered=False) == {"pending": True, "stop_shown": True} + + +def test_queue_that_ends_before_its_start_is_answered_leaves_no_stop_button(): + assert install(queue_ends_before_the_start_is_answered=True) == {"pending": False, "stop_shown": False}