diff --git a/.gitignore b/.gitignore index 096ff35..a5131bc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ __pycache__/ venv/ .env .pytest_cache/ +metrics.txt # Frontend node_modules/ @@ -20,4 +21,3 @@ dist/ # personal tests/ project-plans/ - diff --git a/README.md b/README.md index c3dc4d2..2539f6f 100644 --- a/README.md +++ b/README.md @@ -159,3 +159,22 @@ mongosh --eval "db.runCommand('ping')" } ``` +## Sample Performance Metrics Output + +Below is a real example of metrics collected for uploading 9 forms (each with ~400 questions and 3-10 options per question): +| Metric | Time | Description | +| ------------------------------- | ------------------------------ | ------------------------------------------------ | +| `delete_all_forms_time` | 460ms | Time to delete all forms | +| `deleted_forms` | 9 | Number of forms deleted | +| `deleted_questions` | 3570 | Number of questions deleted | +| `deleted_options` | 23154 | Number of options deleted | +| `validation_time_per_form` | 120-260ms | Time to validate each form file | +| `form_process_time` | 240-1080ms | Time to process and save one form | +| `questions_process_time` | 1.6-2.92s | Time to process and save all questions in a form | +| `avg_one_question_process_time` | 4-7ms | Average time to process one question | +| `options_process_time` | 9.23-11.8s | Time to process and save all options in a form | +| `avg_one_option_process_time` | 3.6-4.5ms | Average time to process one option | +| `total_form_upload_time` | 12.04-15.62s | Total time to process and upload a form | +| `all_forms_batch_process_time` | 15.63s | Time to process all forms in the batch | +| `total_forms` | 9 | Number of forms processed in the batch | +| `avg_one_form_process_time` | 1.74s | Average time to process one form in the batch | diff --git a/backend/README.md b/backend/README.md index 2ea02d0..8d1231f 100644 --- a/backend/README.md +++ b/backend/README.md @@ -102,8 +102,8 @@ The application expects Excel files with three sheets: ## Features -- ✅ Excel file validation with detailed feedback -- ✅ MongoDB integration for data persistence -- ✅ RESTful API endpoints -- ✅ Error handling and logging -- ✅ CORS support for frontend integration \ No newline at end of file +- Excel file validation with detailed feedback +- MongoDB integration for data persistence +- RESTful API endpoints +- Error handling and logging +- CORS support for frontend integration diff --git a/backend/main.py b/backend/main.py index aacb2aa..0f6f1a6 100644 --- a/backend/main.py +++ b/backend/main.py @@ -8,6 +8,8 @@ from database import connect_to_mongo, close_mongo_connection import logging import asyncio from typing import List +import time +import os logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -24,10 +26,22 @@ app.add_middleware( db_service = DatabaseService() +METRICS_FILE = os.path.join(os.path.dirname(__file__), 'metrics.txt') + +def log_metric(metric_name, value): + timestamp = time.strftime('%Y-%m-%d %H:%M:%S') + with open(METRICS_FILE, 'a') as f: + f.write(f"[{timestamp}] {metric_name}: {value}\n") + +startup_time = None + @app.on_event("startup") async def startup_event(): - """Connect to MongoDB on startup""" + """Connect to MongoDB on startup and log cold start time""" + global startup_time + startup_time = time.time() await connect_to_mongo() + log_metric('cold_startup_time', time.strftime('%Y-%m-%d %H:%M:%S')) @app.on_event("shutdown") async def shutdown_event(): @@ -41,7 +55,7 @@ async def validate_file(file: UploadFile): """ if not file.filename or not file.filename.endswith(('.xls', '.xlsx')): return FormValidation( - valid=False, + valid=False, message="Invalid file format. Only .xls/.xlsx files are allowed.", sheets=[], form_metadata={}, @@ -71,7 +85,15 @@ async def upload_files(files: List[UploadFile] = File(...)): logger.error(f"Error processing file {file.filename}: {str(e)}") return {"error": str(e), "filename": file.filename} + batch_start = time.time() results = await asyncio.gather(*(process_file(file) for file in files)) + batch_time = time.time() - batch_start + log_metric('all_forms_batch_process_time', batch_time) + total_forms = len(files) + log_metric('total_forms', total_forms) + if total_forms > 0: + avg_form_time = batch_time / total_forms + log_metric('avg_one_form_process_time', avg_form_time) return results @app.get("/api/forms") @@ -117,11 +139,20 @@ async def delete_form(form_id: str): """ Delete a form and all related data """ + start_delete = time.time() try: + form = await db_service.get_form_by_id(form_id) + questions = await db_service.get_questions_by_form_id(form_id) + options = await db_service.get_options_by_form_id(form_id) + num_questions = len(questions) + num_options = len(options) success = await db_service.delete_form(form_id) + delete_time = time.time() - start_delete + log_metric('delete_form_time', delete_time) + log_metric('deleted_questions', num_questions) + log_metric('deleted_options', num_options) if not success: raise HTTPException(status_code=404, detail="Form not found") - return {"message": "Form deleted successfully"} except HTTPException: raise @@ -134,8 +165,23 @@ async def delete_all_forms(): """ Delete all forms and all related data """ + start_delete = time.time() try: + forms = await db_service.get_all_forms() + questions_count = 0 + options_count = 0 + for form in forms: + questions = await db_service.get_questions_by_form_id(form['id']) + options = await db_service.get_options_by_form_id(form['id']) + questions_count += len(questions) + options_count += len(options) + total_forms = len(forms) success = await db_service.delete_all_forms() + delete_time = time.time() - start_delete + log_metric('delete_all_forms_time', delete_time) + log_metric('deleted_forms', total_forms) + log_metric('deleted_questions', questions_count) + log_metric('deleted_options', options_count) if not success: raise HTTPException(status_code=500, detail="Failed to delete all forms") return {"message": "All forms deleted successfully"} diff --git a/backend/services/xlsform_parser.py b/backend/services/xlsform_parser.py index 72fbd6d..275da6b 100644 --- a/backend/services/xlsform_parser.py +++ b/backend/services/xlsform_parser.py @@ -5,9 +5,18 @@ import uuid from models.form import ParsedForm, FormGroup, Question from services.database_service import DatabaseService import logging +import time +import os logger = logging.getLogger(__name__) +METRICS_FILE = os.path.join(os.path.dirname(__file__), '../metrics.txt') + +def log_metric(metric_name, value): + timestamp = time.strftime('%Y-%m-%d %H:%M:%S') + with open(METRICS_FILE, 'a') as f: + f.write(f"[{timestamp}] {metric_name}: {value}\n") + class XLSFormParser: REQUIRED_SHEETS = ['Forms', 'Questions Info', 'Answer Options'] REQUIRED_FORMS_COLUMNS = ['Language', 'Title'] @@ -18,6 +27,7 @@ class XLSFormParser: self.db_service = DatabaseService() async def validate_file(self, file: UploadFile) -> Dict[str, Any]: + start_validation = time.time() try: df_dict = pd.read_excel(file.file, sheet_name=None) @@ -58,6 +68,8 @@ class XLSFormParser: options_count = len(options_df) is_valid = all(sheet['exists'] and not sheet['missing_columns'] for sheet in sheets_validation) + validation_time = time.time() - start_validation + log_metric('validation_time_per_form', validation_time) return { 'valid': is_valid, @@ -97,6 +109,7 @@ class XLSFormParser: } async def parse_file(self, file: UploadFile) -> ParsedForm: + start_all = time.time() try: df_dict = pd.read_excel(file.file, sheet_name=None) @@ -104,20 +117,37 @@ class XLSFormParser: questions_df = df_dict['Questions Info'] options_df = df_dict['Answer Options'] + start_form = time.time() form_metadata = self._parse_form_metadata(forms_df) - form_id = await self.db_service.save_form(form_metadata) + form_time = time.time() - start_form + log_metric('form_process_time', form_time) + start_questions = time.time() questions_data = self._parse_questions_data(questions_df) question_ids = await self.db_service.save_questions(questions_data, form_id) + questions_time = time.time() - start_questions + log_metric('questions_process_time', questions_time) + if len(questions_data) > 0: + avg_question_time = questions_time / len(questions_data) + log_metric('avg_one_question_process_time', avg_question_time) + start_options = time.time() options_data = self._parse_options_data(options_df) option_ids = await self.db_service.save_options(options_data, form_id) + options_time = time.time() - start_options + log_metric('options_process_time', options_time) + if len(options_data) > 0: + avg_option_time = options_time / len(options_data) + log_metric('avg_one_option_process_time', avg_option_time) form_title = self._get_form_title(forms_df) form_version = '1.0.0' groups = self._parse_questions(questions_df, options_df) + total_time = time.time() - start_all + log_metric('total_form_upload_time', total_time) + return ParsedForm( id=form_id, title=form_title, @@ -128,7 +158,11 @@ class XLSFormParser: 'questions_count': len(questions_data), 'options_count': len(options_data), 'saved_question_ids': question_ids, - 'saved_option_ids': option_ids + 'saved_option_ids': option_ids, + 'form_process_time': form_time, + 'questions_process_time': questions_time, + 'options_process_time': options_time, + 'total_form_upload_time': total_time } ) diff --git a/frontend/README.md b/frontend/README.md index 0036f3b..1da9448 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,59 +1,25 @@ -# MformUpload +# mForm Bulk Upload -This project was generated using [Angular CLI](https://github.com/angular/angular-cli) version 19.2.9. +## About the Frontend -## Development server +This frontend is a modern Angular application for bulk uploading, validating, and managing questionnaire forms (in Excel format). It provides a user-friendly interface for users to drag-and-drop or browse files, validate them, upload them to the backend, and view/manage parsed forms and their details. -To start a local development server, run: +### Main Features +- **Bulk Upload**: Drag-and-drop or select multiple Excel files for upload. +- **Validation**: Validate all selected files before uploading to ensure correct format and content. +- **Form Management**: View a list of all uploaded forms, search by title, and delete individual or all forms. +- **Form Details**: Click on a form to view its questions, options, and metadata in a modal dialog. +- **Responsive UI**: Built with Angular Material for a clean, modern, and responsive user experience. -```bash -ng serve -``` +### Key Components +- **Navbar**: Displays the application title and navigation bar. +- **Search**: Allows searching forms by title. +- **Upload**: Handles file selection, validation, upload, and displays the list of forms and their details. -Once the server is running, open your browser and navigate to `http://localhost:4200/`. The application will automatically reload whenever you modify any of the source files. +### Project Structure +- `src/app/components/navbar/` - Navigation bar component +- `src/app/components/search/` - Search bar component +- `src/app/components/upload/` - Main upload and form management component +- `src/app/services/` - Services for form parsing and backend API communication +- `src/app/models/` - TypeScript interfaces for form and question data -## Code scaffolding - -Angular CLI includes powerful code scaffolding tools. To generate a new component, run: - -```bash -ng generate component component-name -``` - -For a complete list of available schematics (such as `components`, `directives`, or `pipes`), run: - -```bash -ng generate --help -``` - -## Building - -To build the project run: - -```bash -ng build -``` - -This will compile your project and store the build artifacts in the `dist/` directory. By default, the production build optimizes your application for performance and speed. - -## Running unit tests - -To execute unit tests with the [Karma](https://karma-runner.github.io) test runner, use the following command: - -```bash -ng test -``` - -## Running end-to-end tests - -For end-to-end (e2e) testing, run: - -```bash -ng e2e -``` - -Angular CLI does not come with an end-to-end testing framework by default. You can choose one that suits your needs. - -## Additional Resources - -For more information on using the Angular CLI, including detailed command references, visit the [Angular CLI Overview and Command Reference](https://angular.dev/tools/cli) page.