Upload dependency bugfixes & minor improvements

python/upload/__init__.py

@@ -2,6 +2,7 @@ import io
 import os
 import asyncio
 import aiostream
+import json
 from typing import List, Callable
 
 from .spec import FileSpecContextManager, serialize_spec, FileUploadSpec
@@ -29,7 +30,14 @@ async def upload_file_specs(
     for idx, spec in enumerate(serialized_specs):
         spec["id"] = idx
         serialized_specs_dict[idx] = spec
-    
+
+    # Check serialized specs
+    max_size = 50 * 1024 * 1024  # 50 MB in bytes
+    json_data = json.dumps(serialized_specs)
+    json_size = len(json_data.encode('utf-8'))
+    if json_size > max_size:
+        raise Exception(f'Workflow too large error. File specs size exceeds the 50MB limit: {data_size} bytes')
+        
     if hashing_complete_callback is not None:
         hashing_complete_callback()
 
@@ -40,7 +48,6 @@ async def upload_file_specs(
     response_data = await make_post_request(url, { "specs": serialized_specs, "workflow_id": workflow_id })
     upload_data = response_data["data"]
 
-
     # Create the upload data
     # Mix the response data from the server and the file spec
     # and pass it to the generator

python/upload/spec.py

@@ -282,6 +282,10 @@ class _FileSpecContextManager:
         def gen():
             glob = local_path.rglob("*") if recursive else local_path.glob("*")
             for subpath in glob:
+                # skip if subpath contains .git or __pycache__
+                if ".git" in subpath.parts or "__pycache__" in subpath.parts:
+                    continue
+
                 # Skip directories and unsupported file types (e.g. block devices)
                 if subpath.is_file():
                     yield create_file_spec_provider(subpath)