feat: implement CI workflow, enhance file validation, and add test cases

- Introduced a CI workflow for automated testing of backend and frontend components.
- Added pytest configuration for backend tests and Vitest configuration for frontend tests.
- Enhanced file validation logic in the backend to improve error handling and reporting.
- Created comprehensive test cases for various validation scenarios, including edge cases and incorrect formats.
- Updated requirements to include necessary testing libraries and tools.

.github/workflows/ci.yml

@@ -0,0 +1,51 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  backend-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r backend/requirements.txt
+      
+      - name: Run pytest
+        run: |
+          python -m pytest tests/backend -q
+
+  frontend-tests:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: frontend/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run Vitest
+        run: npm run test:run
+

.gitignore

@@ -19,7 +19,6 @@ dist/
 .env/
 
 # personal
-tests/
 project-plans/
 .DS_Store
 .cursorrules

backend/main.py

@@ -1,4 +1,5 @@
 from fastapi import FastAPI, UploadFile, HTTPException, File, Depends
+from starlette.datastructures import UploadFile as StarletteUploadFile
 from fastapi.middleware.cors import CORSMiddleware
 import uvicorn
 from services.xlsform_parser import XLSFormParser
@@ -11,6 +12,7 @@ from typing import List
 import time
 import os
 from fastapi.responses import JSONResponse
+from fastapi import Request
 import pandas as pd
 
 logging.basicConfig(level=logging.INFO)
@@ -51,7 +53,7 @@ async def shutdown_event():
     await close_mongo_connection()
 
 @app.post("/api/validate", response_model=FormValidation)
-async def validate_file(file: UploadFile):
+async def validate_file(file: UploadFile = File(...)):
     """
     Validate the uploaded Excel file format
     """
@@ -74,12 +76,20 @@ async def validate_file(file: UploadFile):
         raise HTTPException(status_code=400, detail=str(e))
 
 @app.post("/api/forms/parse")
-async def parse_file(file: UploadFile):
+async def parse_file(request: Request):
     """
     Parse the uploaded Excel file and return JSON schema without saving to database
     """
-    # Enhanced file validation
-    if not file.filename:
+    form = await request.form()
+    file_field = form.get('file')
+    filename = None
+    upload: UploadFile | None = None
+    if isinstance(file_field, (UploadFile, StarletteUploadFile)):
+        upload = file_field
+        filename = file_field.filename
+    elif isinstance(file_field, str):
+        filename = file_field
+    if not filename:
         raise HTTPException(
             status_code=400,
             detail={
@@ -93,13 +103,13 @@ async def parse_file(file: UploadFile):
             }
         )
     
-    if not file.filename.endswith(('.xls', '.xlsx')):
-        file_extension = file.filename.split('.')[-1] if '.' in file.filename else 'unknown'
+    if not filename.endswith(('.xls', '.xlsx')):
+        file_extension = filename.split('.')[-1] if '.' in filename else 'unknown'
         raise HTTPException(
             status_code=400,
             detail={
                 "error": "Invalid file format",
-                "message": f"File '{file.filename}' has extension '.{file_extension}' but only Excel files (.xls, .xlsx) are supported.",
+                "message": f"File '{filename}' has extension '.{file_extension}' but only Excel files (.xls, .xlsx) are supported.",
                 "error_type": "INVALID_FILE_FORMAT",
                 "received_format": file_extension,
                 "supported_formats": ["xls", "xlsx"],
@@ -113,15 +123,24 @@ async def parse_file(file: UploadFile):
 
     # Enhanced file size validation
     try:
-        file_size = len(await file.read())
-        await file.seek(0)  # Reset file pointer
+        if not upload:
+            raise HTTPException(
+                status_code=400,
+                detail={
+                    "error": "Missing file",
+                    "message": "No file content was uploaded.",
+                    "error_type": "MISSING_FILE"
+                }
+            )
+        file_size = len(await upload.read())
+        await upload.seek(0)
         
         if file_size == 0:
             raise HTTPException(
                 status_code=400,
                 detail={
                     "error": "Empty file",
-                    "message": f"The uploaded file '{file.filename}' is empty (0 bytes).",
+                    "message": f"The uploaded file '{filename}' is empty (0 bytes).",
                     "error_type": "EMPTY_FILE",
                     "file_size": file_size,
                     "suggestions": [
@@ -134,7 +153,7 @@ async def parse_file(file: UploadFile):
         
         # Warn about large files (>10MB)
         if file_size > 10 * 1024 * 1024:
-            logger.warning(f"Large file uploaded: {file.filename} ({file_size} bytes)")
+            logger.warning(f"Large file uploaded: {filename} ({file_size} bytes)")
             
     except Exception as e:
         logger.error(f"Error reading file size: {str(e)}")
@@ -142,7 +161,7 @@ async def parse_file(file: UploadFile):
             status_code=400,
             detail={
                 "error": "File access error",
-                "message": f"Unable to read the uploaded file '{file.filename}'. The file may be corrupted or inaccessible.",
+                "message": f"Unable to read the uploaded file '{filename}'. The file may be corrupted or inaccessible.",
                 "error_type": "FILE_ACCESS_ERROR",
                 "suggestions": [
                     "Try uploading the file again",
@@ -154,7 +173,7 @@ async def parse_file(file: UploadFile):
 
     try:
         parser = XLSFormParser()
-        result = await parser.parse_file_only(file)
+        result = await parser.parse_file_only(upload)
         
         # Check if validation failed
         if isinstance(result, dict) and result.get('valid') == False:
@@ -165,7 +184,7 @@ async def parse_file(file: UploadFile):
                     "error": "Validation failed",
                     "message": result.get('message', 'File validation failed'),
                     "error_type": "VALIDATION_ERROR",
-                    "file_name": result.get('file_name', file.filename),
+                    "file_name": result.get('file_name', filename),
                     "errors": result.get('errors', []),
                     "warnings": result.get('warnings', []),
                     "suggestions": [
@@ -183,14 +202,14 @@ async def parse_file(file: UploadFile):
         raise
     except Exception as e:
         error_message = str(e)
-        logger.error(f"Error parsing file {file.filename}: {error_message}")
+        logger.error(f"Error parsing file {filename}: {error_message}")
         
         # Provide more specific error details based on the exception
         error_detail = {
             "error": "Parsing failed",
-            "message": f"Failed to parse Excel file '{file.filename}': {error_message}",
+            "message": f"Failed to parse Excel file '{filename}': {error_message}",
             "error_type": "PARSING_ERROR",
-            "file_name": file.filename,
+            "file_name": filename,
             "raw_error": error_message,
             "suggestions": [
                 "Check that the Excel file has the required sheets: 'Forms', 'Questions Info', 'Answer Options'",

backend/requirements.txt

@@ -8,3 +8,6 @@ python-dotenv==1.0.0
 motor==3.3.1
 pymongo==4.5.0
 xlrd==2.0.1
+pytest
+pytest-asyncio
+httpx

backend/services/xlsform_parser.py

@@ -871,7 +871,7 @@ class XLSFormParser:
                 'title': form_title,
                 'version': form_version,
                 'language': form_metadata.get('language', 'en'),
-                'groups': [group.dict() for group in groups],
+                'groups': [group.model_dump() for group in groups],
                 'settings': None,
                 'metadata': {
                     'questions_count': len(questions_data),

frontend/package.json

@@ -6,7 +6,9 @@
     "start": "ng serve",
     "build": "ng build",
     "watch": "ng build --watch --configuration development",
-    "serve:ssr:mform-upload": "node --no-deprecation dist/mform-upload/server/server.mjs"
+    "serve:ssr:mform-upload": "node --no-deprecation dist/mform-upload/server/server.mjs",
+    "test": "vitest",
+    "test:run": "vitest run"
   },
   "private": true,
   "dependencies": {
@@ -35,6 +37,9 @@
     "@angular/compiler-cli": "^19.2.0",
     "@types/express": "^4.17.21",
     "@types/node": "^24.0.3",
-    "typescript": "~5.7.2"
+    "typescript": "~5.7.2",
+    "vitest": "^2.0.0",
+    "jsdom": "^25.0.0",
+    "@types/jsdom": "^21.1.7"
   }
 }

frontend/vitest.config.ts

@@ -0,0 +1,21 @@
+import { defineConfig } from 'vitest/config'
+import { fileURLToPath } from 'node:url'
+import { dirname, resolve } from 'node:path'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
+const repoRoot = resolve(__dirname, '..')
+
+export default defineConfig({
+  server: {
+    fs: {
+      allow: ['..']
+    }
+  },
+  test: {
+    environment: 'jsdom',
+    include: [resolve(repoRoot, 'tests/frontend/**/*.spec.ts')],
+    globals: true
+  }
+})
+

pytest.ini

@@ -0,0 +1,7 @@
+[pytest]
+filterwarnings =
+    ignore::DeprecationWarning:fastapi\.openapi\.models
+    ignore::DeprecationWarning:fastapi\.datastructures
+    ignore::DeprecationWarning:pydantic_core\.core_schema
+    ignore:The 'app' shortcut is now deprecated.*:DeprecationWarning:httpx\._client
+    ignore:datetime\.datetime\.utcnow.*:DeprecationWarning:openpyxl\.packaging\.core

tests/backend/conftest.py

@@ -0,0 +1,46 @@
+import os
+import sys
+import pytest
+import pytest_asyncio
+import httpx
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'backend'))
+import main
+
+
+@pytest_asyncio.fixture
+async def client():
+    transport = httpx.ASGITransport(app=main.app)
+    async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as c:
+        yield c
+
+
+@pytest_asyncio.fixture(autouse=True)
+async def mock_db_services(monkeypatch):
+    """Mock database services to avoid requiring MongoDB"""
+    
+    # Mock database connection functions
+    async def mock_connect_to_mongo():
+        pass
+    
+    async def mock_close_mongo_connection():
+        pass
+    
+    monkeypatch.setattr(main, 'connect_to_mongo', mock_connect_to_mongo)
+    monkeypatch.setattr(main, 'close_mongo_connection', mock_close_mongo_connection)
+    
+    # Mock database service
+    class MockDatabaseService:
+        async def get_all_forms(self):
+            return []
+        
+        async def get_form_by_id(self, form_id: str):
+            return None  # Simulate form not found
+        
+        async def get_questions_by_form_id(self, form_id: str):
+            return []
+        
+        async def get_options_by_form_id(self, form_id: str):
+            return []
+    
+    monkeypatch.setattr(main, 'db_service', MockDatabaseService())

tests/backend/test_api.py

@@ -0,0 +1,97 @@
+import os
+import sys
+import io
+import pytest
+import httpx
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'backend'))
+import main
+
+
+@pytest.mark.asyncio
+async def test_validate_invalid_extension(client: httpx.AsyncClient):
+    """Test validation endpoint with invalid file extension"""
+    files = {'file': ('bad.txt', b'data', 'text/plain')}
+    resp = await client.post('/api/validate', files=files)
+    assert resp.status_code == 200
+    body = resp.json()
+    assert body['valid'] is False
+    assert 'Only .xls/.xlsx' in body['message']
+
+
+@pytest.mark.asyncio
+async def test_parse_missing_file_name(client: httpx.AsyncClient):
+    """Test parse endpoint with no file uploaded"""
+    resp = await client.post('/api/forms/parse', data={})
+    assert resp.status_code == 400
+    detail = resp.json()['detail']
+    assert detail['error_type'] == 'MISSING_FILE'
+
+
+@pytest.mark.asyncio
+async def test_parse_invalid_extension(client: httpx.AsyncClient):
+    """Test parse endpoint with invalid file extension"""
+    files = {'file': ('bad.txt', io.BytesIO(b'data'), 'text/plain')}
+    resp = await client.post('/api/forms/parse', files=files)
+    assert resp.status_code == 400
+    detail = resp.json()['detail']
+    assert detail['error_type'] == 'INVALID_FILE_FORMAT'
+
+
+@pytest.mark.asyncio
+async def test_get_all_forms_empty(client: httpx.AsyncClient):
+    """Test getting all forms when database is empty (mocked)"""
+    resp = await client.get('/api/forms')
+    assert resp.status_code == 200
+    body = resp.json()
+    assert 'forms' in body
+    assert 'count' in body
+    assert body['count'] == 0
+    assert body['forms'] == []
+
+
+@pytest.mark.asyncio
+async def test_get_form_by_id_not_found(client: httpx.AsyncClient):
+    """Test getting a form by ID when form doesn't exist (mocked)"""
+    resp = await client.get('/api/forms/507f1f77bcf86cd799439011')  # Valid ObjectId format
+    assert resp.status_code == 404
+
+
+@pytest.mark.asyncio
+async def test_parse_with_valid_excel_file(client: httpx.AsyncClient):
+    """Test parsing a valid Excel file using local test files"""
+    test_file_path = os.path.join(os.path.dirname(__file__), '..', 'test_xlsforms_valid', 'valid_form_1.xlsx')
+    
+    if os.path.exists(test_file_path):
+        with open(test_file_path, 'rb') as f:
+            files = {'file': ('valid_form_1.xlsx', f, 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')}
+            resp = await client.post('/api/forms/parse', files=files)
+            # Should return parsed form data without saving to DB
+            assert resp.status_code == 200
+            body = resp.json()
+            assert 'title' in body
+            assert 'groups' in body
+            assert 'metadata' in body
+    else:
+        # Skip test if no test file available
+        pytest.skip("Test Excel file not found")
+
+
+@pytest.mark.asyncio
+async def test_validate_with_valid_excel_file(client: httpx.AsyncClient):
+    """Test validation with a valid Excel file using local test files"""
+    test_file_path = os.path.join(os.path.dirname(__file__), '..', 'test_xlsforms_valid', 'valid_form_1.xlsx')
+    
+    if os.path.exists(test_file_path):
+        with open(test_file_path, 'rb') as f:
+            files = {'file': ('valid_form_1.xlsx', f, 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet')}
+            resp = await client.post('/api/validate', files=files)
+            assert resp.status_code == 200
+            body = resp.json()
+            assert 'valid' in body
+            assert 'message' in body
+            assert 'sheets' in body
+    else:
+        # Skip test if no test file available
+        pytest.skip("Test Excel file not found")
+

tests/codeshares

@@ -0,0 +1,5 @@
+https://codeshare.io/5vwmgl
+https://codeshare.io/5eVpNA
+https://codeshare.io/GLwbe7
+https://codeshare.io/ar9xeq
+https://codeshare.io/GAdOBE

tests/frontend/form.service.spec.ts

@@ -0,0 +1,59 @@
+import type { HttpClient } from '@angular/common/http'
+import { describe, it, expect, vi } from 'vitest'
+
+function of<T>(value: T) {
+  return {
+    subscribe: (next?: (v: T) => void) => {
+      if (typeof next === 'function') next(value)
+      return { unsubscribe() {} }
+    }
+  } as any
+}
+import { FormService } from '../../frontend/src/app/services/form.service'
+
+class HttpClientStub {
+  post<T>(url: string, body: any) {
+    return of({} as T)
+  }
+  get<T>(url: string) {
+    return of({} as T)
+  }
+  delete<T>(url: string) {
+    return of({} as T)
+  }
+  put<T>(url: string, body: any) {
+    return of({} as T)
+  }
+}
+
+describe('FormService', () => {
+  let service: FormService
+
+  beforeEach(() => {
+    service = new FormService(new HttpClientStub() as unknown as HttpClient)
+  })
+
+  it('should call validate endpoint', async () => {
+    const file = new Blob(['a']) as any
+    const spy = vi.spyOn(HttpClientStub.prototype, 'post')
+    service.validateFile(file).subscribe()
+    expect(spy).toHaveBeenCalled()
+    expect(spy.mock.calls[0][0]).toContain('/api/validate')
+  })
+
+  it('should call parse endpoint', async () => {
+    const file = new Blob(['a']) as any
+    const spy = vi.spyOn(HttpClientStub.prototype, 'post')
+    service.parseFile(file).subscribe()
+    expect(spy).toHaveBeenCalled()
+    expect(spy.mock.calls[0][0]).toContain('/api/forms/parse')
+  })
+
+  it('should call getAllForms endpoint', async () => {
+    const spy = vi.spyOn(HttpClientStub.prototype, 'get')
+    service.getAllForms().subscribe()
+    expect(spy).toHaveBeenCalled()
+    expect(spy.mock.calls[0][0]).toContain('/api/forms')
+  })
+})
+

tests/make-valid.py

@@ -0,0 +1,80 @@
+import pandas as pd
+import os
+from faker import Faker
+import random
+
+fake = Faker()
+output_dir = "test_xlsforms_valid"  # Changed directory name
+os.makedirs(output_dir, exist_ok=True)
+
+VALID_INPUT_TYPES = [1, 2, 3, 4, 5, 6, 7]  # Example: 1=Text, 2=Select One, 3=Select Many, etc.
+
+def generate_valid_xlsform(file_name_prefix, num_questions, num_options_per_question):
+    forms_data = {
+        "Language": [random.choice(["en", "fr", "es", "de"])],
+        "Title": [fake.catch_phrase() + " Form"]
+    }
+    forms_df = pd.DataFrame(forms_data)
+
+    questions_data = []
+    for i in range(num_questions):
+        question_order = i + 1
+        question_type = random.choice(VALID_INPUT_TYPES)
+        questions_data.append({
+            "Order": question_order,
+            "Title": fake.sentence(nb_words=6),
+            "View Sequence": i + 1,
+            "Input Type": question_type,
+            "Optional Column 1": fake.word(), # Add some optional columns
+            "Optional Column 2": fake.random_int(min=1, max=100)
+        })
+    questions_df = pd.DataFrame(questions_data)
+
+    options_data = []
+    for question_idx, question in questions_df.iterrows():
+        if question["Input Type"] in [2, 3]:  # Select One or Select Many
+            # Generate 3-10 options per question
+            num_options_for_question = random.randint(3, 10)
+            for i in range(num_options_for_question):
+                # Generate a more robust label that won't be empty
+                label = fake.catch_phrase().split()[0].capitalize()
+                if not label or len(label) < 2:
+                    label = f"Option{i+1}"
+
+                options_data.append({
+                    "Order": question["Order"],
+                    "Id": i + 1,
+                    "Label": label
+                })
+    options_df = pd.DataFrame(options_data)
+
+    # Validate that all labels are non-empty
+    if not options_df.empty:
+        # Replace any empty or null labels with fallback values
+        options_df['Label'] = options_df['Label'].fillna('Default Option')
+        options_df.loc[options_df['Label'] == '', 'Label'] = 'Default Option'
+        options_df.loc[options_df['Label'].str.len() < 2, 'Label'] = 'Default Option'
+
+    file_path = os.path.join(output_dir, f"{file_name_prefix}.xlsx")
+    with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+        forms_df.to_excel(writer, sheet_name="Forms", index=False)
+        questions_df.to_excel(writer, sheet_name="Questions Info", index=False)
+        options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+    return file_path
+
+# Generate 9 forms with ~400 questions each
+num_files_to_generate = 100
+generated_files = []
+
+for i in range(num_files_to_generate):
+    file_name = f"valid_form_{i+1}"
+    # Generate approximately 400 questions per form (with some variation)
+    num_questions = random.randint(380, 420)  # ~400 questions with ±20 variation
+
+    path = generate_valid_xlsform(file_name, num_questions, 10)  # Max 10 options, actual will be 3-10
+    generated_files.append(path)
+    print(f"Generated form {i+1}: {num_questions} questions")
+
+print(f"\nSuccessfully generated {len(generated_files)} valid test files in '{output_dir}'")
+print("Each form contains approximately 400 questions with 3-10 options per select question.")

tests/make.py

@@ -0,0 +1,273 @@
+import pandas as pd
+import os
+from faker import Faker
+import random
+
+fake = Faker()
+output_dir = "test_xlsforms_valid"
+os.makedirs(output_dir, exist_ok=True)
+
+forms_df = pd.DataFrame([{
+    "Language": "en",
+    "Title": "Test Form Missing Forms Sheet"
+}])
+
+questions_df = pd.DataFrame([{
+    "Order": 1,
+    "Title": "Sample Question",
+    "View Sequence": 1,
+    "Input Type": 1
+}])
+
+options_df = pd.DataFrame([{
+    "Order": 1,
+    "Id": 1,
+    "Label": "Option 1"
+}])
+
+file_path = os.path.join(output_dir, "missing_forms_sheet.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    # Skip the Forms sheet intentionally
+    questions_df.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+file_path = os.path.join(output_dir, "missing_questions_sheet.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_df.to_excel(writer, sheet_name="Forms", index=False)
+    # Skip the Questions Info sheet intentionally
+    options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+file_path = os.path.join(output_dir, "missing_options_sheet.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_df.to_excel(writer, sheet_name="Forms", index=False)
+    questions_df.to_excel(writer, sheet_name="Questions Info", index=False)
+    # Skip the Answer Options sheet intentionally
+
+forms_missing_lang = pd.DataFrame([{
+    "Title": "Form with missing Language column"
+}])
+
+file_path = os.path.join(output_dir, "missing_language_column.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_missing_lang.to_excel(writer, sheet_name="Forms", index=False)
+    questions_df.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+forms_missing_title = pd.DataFrame([{
+    "Language": "en"
+}])
+
+file_path = os.path.join(output_dir, "missing_title_column.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_missing_title.to_excel(writer, sheet_name="Forms", index=False)
+    questions_df.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+questions_missing_order = pd.DataFrame([{
+    "Title": "Question without Order",
+    "View Sequence": 1,
+    "Input Type": 1
+}])
+
+file_path = os.path.join(output_dir, "missing_order_column.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_df.to_excel(writer, sheet_name="Forms", index=False)
+    questions_missing_order.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+questions_invalid_types = pd.DataFrame([
+    {"Order": 1, "Title": "Valid Question", "View Sequence": 1, "Input Type": 1},
+    {"Order": 2, "Title": "Invalid Type 99", "View Sequence": 2, "Input Type": 99},
+    {"Order": 3, "Title": "Invalid Type Text", "View Sequence": 3, "Input Type": "text"},
+    {"Order": 4, "Title": "Invalid Type -1", "View Sequence": 4, "Input Type": -1},
+])
+
+file_path = os.path.join(output_dir, "invalid_question_types.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_df.to_excel(writer, sheet_name="Forms", index=False)
+    questions_invalid_types.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+questions_invalid_order = pd.DataFrame([
+    {"Order": "first", "Title": "Question with text order", "View Sequence": 1, "Input Type": 1},
+    {"Order": 2.5, "Title": "Question with decimal order", "View Sequence": 2, "Input Type": 1},
+    {"Order": -1, "Title": "Question with negative order", "View Sequence": 3, "Input Type": 1},
+    {"Order": 0, "Title": "Question with zero order", "View Sequence": 4, "Input Type": 1},
+])
+
+file_path = os.path.join(output_dir, "invalid_order_values.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_df.to_excel(writer, sheet_name="Forms", index=False)
+    questions_invalid_order.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+questions_duplicate_order = pd.DataFrame([
+    {"Order": 1, "Title": "First question", "View Sequence": 1, "Input Type": 1},
+    {"Order": 1, "Title": "Duplicate order question", "View Sequence": 2, "Input Type": 2},
+    {"Order": 2, "Title": "Third question", "View Sequence": 3, "Input Type": 1},
+])
+
+file_path = os.path.join(output_dir, "duplicate_question_order.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_df.to_excel(writer, sheet_name="Forms", index=False)
+    questions_duplicate_order.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+options_duplicate = pd.DataFrame([
+    {"Order": 1, "Id": 1, "Label": "Option 1"},
+    {"Order": 1, "Id": 1, "Label": "Duplicate Option 1"},  # Duplicate combination
+    {"Order": 1, "Id": 2, "Label": "Option 2"},
+    {"Order": 2, "Id": 1, "Label": "Question 2 Option 1"},
+])
+
+file_path = os.path.join(output_dir, "duplicate_option_ids.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_df.to_excel(writer, sheet_name="Forms", index=False)
+    questions_df.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_duplicate.to_excel(writer, sheet_name="Answer Options", index=False)
+
+forms_empty = pd.DataFrame([{
+    "Language": "",
+    "Title": ""
+}])
+
+file_path = os.path.join(output_dir, "empty_form_values.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_empty.to_excel(writer, sheet_name="Forms", index=False)
+    questions_df.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+questions_missing_titles = pd.DataFrame([
+    {"Order": 1, "Title": "Valid question", "View Sequence": 1, "Input Type": 1},
+    {"Order": 2, "Title": "", "View Sequence": 2, "Input Type": 2},
+    {"Order": 3, "Title": None, "View Sequence": 3, "Input Type": 1},
+])
+
+file_path = os.path.join(output_dir, "missing_question_titles.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_df.to_excel(writer, sheet_name="Forms", index=False)
+    questions_missing_titles.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+options_missing_labels = pd.DataFrame([
+    {"Order": 1, "Id": 1, "Label": "Valid option"},
+    {"Order": 1, "Id": 2, "Label": ""},
+    {"Order": 1, "Id": 3, "Label": None},
+])
+
+file_path = os.path.join(output_dir, "missing_option_labels.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_df.to_excel(writer, sheet_name="Forms", index=False)
+    questions_df.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_missing_labels.to_excel(writer, sheet_name="Answer Options", index=False)
+
+questions_choice_no_options = pd.DataFrame([
+    {"Order": 1, "Title": "Text question", "View Sequence": 1, "Input Type": 1},
+    {"Order": 2, "Title": "Select one without options", "View Sequence": 2, "Input Type": 2},
+    {"Order": 3, "Title": "Select multiple without options", "View Sequence": 3, "Input Type": 3},
+])
+
+options_limited = pd.DataFrame([
+    {"Order": 1, "Id": 1, "Label": "Option for text question (shouldn't be here)"}
+])
+
+file_path = os.path.join(output_dir, "choice_questions_no_options.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_df.to_excel(writer, sheet_name="Forms", index=False)
+    questions_choice_no_options.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_limited.to_excel(writer, sheet_name="Answer Options", index=False)
+
+questions_limited = pd.DataFrame([
+    {"Order": 1, "Title": "Only question", "View Sequence": 1, "Input Type": 1},
+])
+
+options_orphaned = pd.DataFrame([
+    {"Order": 1, "Id": 1, "Label": "Option for existing question"},
+    {"Order": 2, "Id": 1, "Label": "Option for non-existent question"},
+    {"Order": 3, "Id": 1, "Label": "Another orphaned option"},
+])
+
+file_path = os.path.join(output_dir, "orphaned_options.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_df.to_excel(writer, sheet_name="Forms", index=False)
+    questions_limited.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_orphaned.to_excel(writer, sheet_name="Answer Options", index=False)
+
+forms_invalid_lang = pd.DataFrame([
+    {"Language": "invalid_lang", "Title": "Form with invalid language code"}
+])
+
+file_path = os.path.join(output_dir, "invalid_language_code.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_invalid_lang.to_excel(writer, sheet_name="Forms", index=False)
+    questions_df.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+forms_long_title = pd.DataFrame([{
+    "Language": "en",
+    "Title": "A" * 300  # Very long title (over 255 chars)
+}])
+
+questions_long_title = pd.DataFrame([
+    {"Order": 1, "Title": "B" * 1200, "View Sequence": 1, "Input Type": 1},  # Very long question
+])
+
+options_long_label = pd.DataFrame([
+    {"Order": 1, "Id": 1, "Label": "C" * 600},  # Very long option label
+])
+
+file_path = os.path.join(output_dir, "very_long_values.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_long_title.to_excel(writer, sheet_name="Forms", index=False)
+    questions_long_title.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_long_label.to_excel(writer, sheet_name="Answer Options", index=False)
+
+forms_multiple_issues = pd.DataFrame([
+    {"Language": "", "Title": ""},  # Missing both required values
+    {"Language": "fr", "Title": "Extra row (should cause warning)"}  # Extra row
+])
+
+questions_multiple_issues = pd.DataFrame([
+    {"Order": "bad", "Title": "", "View Sequence": -1, "Input Type": 99},  # Multiple errors
+    {"Order": 1, "Title": "Valid question", "View Sequence": 1, "Input Type": 2},  # Choice question
+    {"Order": 1, "Title": "Duplicate order", "View Sequence": 2, "Input Type": 1},  # Duplicate order
+])
+
+options_multiple_issues = pd.DataFrame([
+    {"Order": "bad", "Id": "bad", "Label": ""},  # All bad values
+    {"Order": 2, "Id": 1, "Label": "Option for non-choice question"},  # Orphaned
+])
+
+file_path = os.path.join(output_dir, "multiple_issues_complex.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    forms_multiple_issues.to_excel(writer, sheet_name="Forms", index=False)
+    questions_multiple_issues.to_excel(writer, sheet_name="Questions Info", index=False)
+    options_multiple_issues.to_excel(writer, sheet_name="Answer Options", index=False)
+
+empty_df = pd.DataFrame()
+
+file_path = os.path.join(output_dir, "empty_sheets.xlsx")
+with pd.ExcelWriter(file_path, engine="openpyxl") as writer:
+    empty_df.to_excel(writer, sheet_name="Forms", index=False)
+    empty_df.to_excel(writer, sheet_name="Questions Info", index=False)
+    empty_df.to_excel(writer, sheet_name="Answer Options", index=False)
+
+with open(os.path.join(output_dir, "fake_excel.xlsx"), 'w') as f:
+    f.write("This is not an Excel file")
+
+csv_content = "Language,Title\nen,Fake Excel File"
+with open(os.path.join(output_dir, "csv_as_xlsx.xlsx"), 'w') as f:
+    f.write(csv_content)
+
+print(f"\nSuccessfully generated {len(os.listdir(output_dir))} incorrect test files in '{output_dir}'")
+print("\nTest file categories created:")
+print("• Missing sheets (3 files)")
+print("• Missing columns (3 files)")
+print("• Invalid data types (2 files)")
+print("• Duplicate values (2 files)")
+print("• Missing required values (3 files)")
+print("• Cross-reference issues (2 files)")
+print("• Language/format issues (2 files)")
+print("• Complex edge cases (2 files)")
+print("• Invalid file formats (2 files)")
+

tests/test_validation.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""
+Test script to demonstrate the enhanced validation system
+Usage: python test_validation.py
+"""
+
+import asyncio
+import sys
+import os
+import io
+import json
+
+# Add backend to path
+backend_path = os.path.join(os.path.dirname(__file__), '..', 'backend')
+sys.path.insert(0, backend_path)
+
+try:
+    from fastapi import UploadFile
+    from services.xlsform_parser import XLSFormParser
+except ImportError as e:
+    print(f"❌ Import error: {e}")
+    print("Make sure you're running this from the tests directory and have the required dependencies installed.")
+    sys.exit(1)
+
+async def test_validation_file(file_path: str):
+    """Test validation for a single file"""
+    print(f"\n🧪 Testing: {os.path.basename(file_path)}")
+    print("=" * 60)
+
+    try:
+        # Read the file
+        with open(file_path, 'rb') as f:
+            file_content = f.read()
+
+        # Create UploadFile object
+        upload_file = UploadFile(
+            filename=os.path.basename(file_path),
+            file=io.BytesIO(file_content)
+        )
+
+        # Initialize parser and validate
+        parser = XLSFormParser()
+        result = await parser.validate_file(upload_file)
+
+        # Display results
+        print(f"✅ Valid: {result['valid']}")
+        print(f"📝 Message: {result['message']}")
+
+        if result.get('errors'):
+            print(f"\n❌ Errors ({len(result['errors'])}):")
+            for error in result['errors']:
+                location = f"{error['location']}"
+                if error.get('row'):
+                    location += f" (Row {error['row']})"
+                if error.get('column'):
+                    location += f" (Column: {error['column']})"
+                print(f"  • {error['type']}: {error['message']} [{location}]")
+
+        if result.get('warnings'):
+            print(f"\n⚠️  Warnings ({len(result['warnings'])}):")
+            for warning in result['warnings']:
+                location = f"{warning['location']}"
+                if warning.get('row'):
+                    location += f" (Row {warning['row']})"
+                if warning.get('column'):
+                    location += f" (Column: {warning['column']})"
+                print(f"  • {warning['type']}: {warning['message']} [{location}]")
+
+        # Sheet validation details
+        if result.get('sheets'):
+            print(f"\n📊 Sheet Validation:")
+            for sheet in result['sheets']:
+                status = "✅" if sheet['exists'] and not sheet['missing_columns'] else "❌"
+                print(f"  {status} {sheet['name']}: {sheet['row_count']} rows")
+                if sheet['missing_columns']:
+                    print(f"    Missing columns: {', '.join(sheet['missing_columns'])}")
+
+        return result['valid']
+
+    except Exception as e:
+        print(f"❌ Error testing file: {str(e)}")
+        return False
+
+async def main():
+    """Main test function"""
+    print("🚀 Enhanced XLSForm Validation Test Suite")
+    print("=" * 60)
+
+    # Test directory
+    test_dir = os.path.join(os.path.dirname(__file__), 'test_xlsforms_incorrect')
+
+    if not os.path.exists(test_dir):
+        print(f"❌ Test directory not found: {test_dir}")
+        print("Please run 'python make.py' first to generate test files.")
+        return
+
+    # Get all test files
+    test_files = [f for f in os.listdir(test_dir) if f.endswith('.xlsx')]
+
+    if not test_files:
+        print(f"❌ No test files found in {test_dir}")
+        return
+
+    print(f"📁 Found {len(test_files)} test files")
+
+    # Test each file
+    valid_count = 0
+    total_count = len(test_files)
+
+    for test_file in sorted(test_files):
+        file_path = os.path.join(test_dir, test_file)
+        is_valid = await test_validation_file(file_path)
+        if is_valid:
+            valid_count += 1
+
+    # Summary
+    print("\n" + "=" * 60)
+    print("📈 Test Summary")
+    print("=" * 60)
+    print(f"Total files tested: {total_count}")
+    print(f"Valid files: {valid_count}")
+    print(f"Invalid files: {total_count - valid_count}")
+    print(f"Success rate: {(total_count - valid_count) / total_count * 100:.1f}% (lower is better for error testing)")
+
+    if valid_count > 0:
+        print(f"\n⚠️  Warning: {valid_count} files passed validation when they should have failed!")
+        print("This may indicate issues with the validation logic.")
+    else:
+        print(f"\n🎉 Excellent! All test files correctly failed validation.")
+        print("The enhanced validation system is working properly!")
+
+if __name__ == "__main__":
+    asyncio.run(main())

tests/test_xlsforms_incorrect/csv_as_xlsx.xlsx

@@ -0,0 +1,2 @@
+Language,Title
+en,Fake Excel File

tests/test_xlsforms_incorrect/fake_excel.xlsx

@@ -0,0 +1 @@
+This is not an Excel file