add a plugin schema test in pytest

test/plugins/plugin.schema.json

@@ -0,0 +1,53 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "name": { "type": "string" },
+    "uniqueId": { "type": "string" },
+    "description": { "type": "string" },
+    "plugin-format": { "type": "string" },
+    "type": {
+      "type": "string",
+      "enum": [
+        "loader",
+        "trainer",
+        "evaluator",
+        "generator",
+        "eval",
+        "exporter",
+        "rag"
+      ]
+    },
+    "version": { "type": "string" },
+    "model_architectures": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "supported_hardware_architectures": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "enum": ["cuda", "mlx", "cpu"]
+      }
+    },
+    "files": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "setup-script": { "type": "string" },
+    "parameters": { "type": "object" },
+    "parameters_ui": { "type": "object" }
+  },
+  "required": [
+    "name",
+    "uniqueId",
+    "description",
+    "plugin-format",
+    "type",
+    "version",
+    "files",
+    "setup-script",
+    "supported_hardware_architectures"
+  ],
+  "additionalProperties": true
+}

test/plugins/test_plugins_schema.py

@@ -0,0 +1,37 @@
+import os
+import json
+import pytest
+from jsonschema import validate, ValidationError
+
+# Get the directory of the current script
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Load the schema using a relative path
+SCHEMA_PATH = os.path.join(BASE_DIR, "plugin.schema.json")
+with open(SCHEMA_PATH, "r") as schema_file:
+    SCHEMA = json.load(schema_file)
+
+# Define the base directory for plugins using a relative path
+PLUGINS_DIR = os.path.join(BASE_DIR, "../../transformerlab/plugins")
+
+
+def find_index_json_files(base_dir):
+    """Find all index.json files in the first level of each folder under the given directory."""
+    for entry in os.scandir(base_dir):
+        if entry.is_dir():
+            index_file = os.path.join(entry.path, "index.json")
+            if os.path.isfile(index_file):
+                yield index_file
+
+
+@pytest.mark.parametrize("index_json_path", find_index_json_files(PLUGINS_DIR))
+def test_validate_index_json(index_json_path):
+    """Validate each index.json file against the schema."""
+    with open(index_json_path, "r") as json_file:
+        try:
+            data = json.load(json_file)
+            validate(instance=data, schema=SCHEMA)
+        except ValidationError as e:
+            pytest.fail(f"Validation failed for {index_json_path}: {e.message}")
+        except json.JSONDecodeError as e:
+            pytest.fail(f"Invalid JSON in {index_json_path}: {e.msg}")

transformerlab/plugins/airllm_mlx_server/index.json

@@ -20,13 +20,8 @@
     "Phi3ForCausalLM",
     "Qwen2ForCausalLM"
   ],
-  "supported_hardware_architectures": [
+  "supported_hardware_architectures": ["mlx"],
-    "mlx"
+  "files": ["main.py", "setup.sh"],
-  ],
-  "files": [
-    "main.py",
-    "setup.sh"
-  ],
   "setup-script": "setup.sh",
-  "parameters": []
+  "parameters": {}
-}
+}

transformerlab/plugins/mlx_server/index.json

@@ -23,5 +23,5 @@
   "supported_hardware_architectures": ["mlx"],
   "files": ["main.py", "setup.sh"],
   "setup-script": "setup.sh",
-  "parameters": []
+  "parameters": {}
 }

transformerlab/plugins/mlx_vlm_server/index.json

@@ -5,13 +5,8 @@
   "plugin-format": "python",
   "type": "loader",
   "version": "0.1.18",
-  "model_architectures": [
+  "model_architectures": ["LlavaForConditionalGeneration"],
-    "LlavaForConditionalGeneration"
+  "files": ["main.py", "setup.sh"],
-  ],
-  "files": [
-    "main.py",
-    "setup.sh"
-  ],
   "setup-script": "setup.sh",
-  "parameters": []
+  "parameters": {}
-}
+}