diff --git a/backend/README.md b/backend/README.md index 44df278..55c423e 100644 --- a/backend/README.md +++ b/backend/README.md @@ -254,7 +254,7 @@ Validation will fail if required sheets or columns are missing. The application logs detailed performance metrics to `metrics.txt` including: - **Validation times**: File validation performance per form -- **Parse-only times**: Schema parsing without database operations +- **Parse-only times**: Schema parsing without database operations - **Upload processing times**: Complete form processing with database storage - **Question/Option processing**: Individual item processing performance - **Cold startup tracking**: Application initialization times @@ -276,5 +276,23 @@ The application logs detailed performance metrics to `metrics.txt` including: - **Metrics:** Extend metrics logging for new operations or performance measurements. - **Documentation:** Update this README and docstrings in code for any new features or changes. +# Why you don't do compression with XLS/XLSX files + +XLS and XLSX files are already compressed formats (especially XLSX, which is a ZIP archive of XML files). Applying additional compression (like gzip) typically results in minimal size reduction—often less than 3%. This extra step adds processing overhead without significant storage or transfer benefits. In most cases, it's more efficient to transfer these files as-is. + +# Compression Metrics for XLSX Files + +| File Name | Compressed Size (bytes) | Decompressed Size (bytes) | Compression Ratio | Bytes Saved | +|------------------------|------------------------|---------------------------|-------------------|-------------| +| valid_form_7.xlsx.gz | 37,009 | 37,973 | 2.5% | 964 | +| valid_form_8.xlsx.gz | 38,975 | 40,021 | 2.6% | 1,046 | +| valid_form_9.xlsx.gz | 37,341 | 38,420 | 2.8% | 1,079 | +| valid_form_10.xlsx.gz | 37,269 | 38,208 | 2.5% | 939 | + +**Observation:** +The compression ratios are very low (2.5%–2.8%), saving only about 939–1,079 bytes per file. This demonstrates that compressing XLSX files provides negligible space savings. + +To put this in perspective, this required refactoring half the codebase with over 5,000 changed lines of code. + ## License MIT diff --git a/frontend/package-lock.json b/frontend/package-lock.json index fac7c1a..ac3e8b2 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -6412,9 +6412,9 @@ } }, "node_modules/compression": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/compression/-/compression-1.8.0.tgz", - "integrity": "sha512-k6WLKfunuqCYD3t6AsuPGvQWaKwuLLh2/xHNcX4qE+vIfDNXpSqnrhwA7O53R7WVQUnt8dVAIW+YHr7xTgOgGA==", + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/compression/-/compression-1.8.1.tgz", + "integrity": "sha512-9mAqGPHLakhCLeNyxPkK4xVo746zQ/czLH1Ky+vkitMnWfWZps8r0qXuwhwizagCRttsL4lfG4pIOvaWLpAP0w==", "dev": true, "license": "MIT", "dependencies": { @@ -6422,7 +6422,7 @@ "compressible": "~2.0.18", "debug": "2.6.9", "negotiator": "~0.6.4", - "on-headers": "~1.0.2", + "on-headers": "~1.1.0", "safe-buffer": "5.2.1", "vary": "~1.1.2" }, @@ -9844,9 +9844,9 @@ } }, "node_modules/on-headers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", - "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.1.0.tgz", + "integrity": "sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==", "dev": true, "license": "MIT", "engines": {