# Performance-optimized data analysis workflow
class PerformantDataAnalyzer:
    def __init__(self):
        self.chunk_size = 10000
        self.memory_threshold = 2 * 1024 * 1024 * 1024  # 2GB
        self.performance_metrics = {}
    
    async def analyze_large_dataset(self, file_path, analysis_config):
        """Analyze large datasets with performance optimization."""
        
        import time
        import psutil
        
        start_time = time.time()
        initial_memory = psutil.Process().memory_info().rss
        
        try:
            # Check file size to determine processing strategy
            file_size = await self.get_file_size(file_path)
            
            if file_size > self.memory_threshold:
                return await self.chunked_analysis(file_path, analysis_config)
            else:
                return await self.standard_analysis(file_path, analysis_config)
                
        finally:
            # Record performance metrics
            end_time = time.time()
            final_memory = psutil.Process().memory_info().rss
            
            self.performance_metrics = {
                'execution_time': end_time - start_time,
                'memory_usage': final_memory - initial_memory,
                'file_size': file_size,
                'processing_method': 'chunked' if file_size > self.memory_threshold else 'standard'
            }
    
    async def chunked_analysis(self, file_path, config):
        """Process large files in chunks."""
        
        results = []
        chunk_count = 0
        
        async for chunk in self.read_file_chunks(file_path):
            chunk_result = await self.analyze_chunk(chunk, config)
            results.append(chunk_result)
            chunk_count += 1
            
            print(f"📊 Processed chunk {chunk_count} ({len(chunk)} rows)")
            
            # Monitor memory usage
            memory_usage = psutil.Process().memory_info().rss
            if memory_usage > self.memory_threshold * 0.8:
                print("⚠️ High memory usage detected, optimizing...")
                await self.optimize_memory()
        
        # Combine chunk results
        return await self.combine_chunk_results(results, config)
    
    async def standard_analysis(self, file_path, config):
        """Standard analysis for smaller files."""
        
        if file_path.endswith('.xlsx'):
            return await xlsxAnalysis({
                'filePath': file_path,
                **config
            })
        else:
            return await statisticalAnalysis({
                'data': file_path,
                **config
            })
    
    async def read_file_chunks(self, file_path):
        """Generator for reading file in chunks."""
        
        import pandas as pd
        
        if file_path.endswith('.csv'):
            for chunk in pd.read_csv(file_path, chunksize=self.chunk_size):
                yield chunk
        elif file_path.endswith('.xlsx'):
            # For Excel files, read sheet by sheet
            xl_file = pd.ExcelFile(file_path)
            for sheet_name in xl_file.sheet_names:
                chunk = pd.read_excel(file_path, sheet_name=sheet_name)
                yield chunk
    
    async def analyze_chunk(self, chunk, config):
        """Analyze individual chunk."""
        
        # Convert chunk to temporary file for processing
        temp_path = f'/sandbox/temp/chunk_{id(chunk)}.csv'
        chunk.to_csv(temp_path, index=False)
        
        try:
            result = await statisticalAnalysis({
                'data': temp_path,
                **config
            })
            return result
        finally:
            # Clean up temporary file
            import os
            if os.path.exists(temp_path):
                os.remove(temp_path)
    
    async def combine_chunk_results(self, chunk_results, config):
        """Combine results from chunk processing."""
        
        if config.get('analysis') == 'descriptive_statistics':
            return self.combine_statistical_results(chunk_results)
        elif config.get('operation') == 'aggregation':
            return self.combine_aggregation_results(chunk_results)
        else:
            return {'chunk_results': chunk_results}
    
    def combine_statistical_results(self, results):
        """Combine statistical analysis results."""
        
        combined = {
            'total_records': sum(r.get('record_count', 0) for r in results),
            'summary_statistics': {},
            'chunks_processed': len(results)
        }
        
        # Combine statistics (weighted by chunk size)
        for column in results[0].get('columns', []):
            combined['summary_statistics'][column] = self.weighted_statistics(
                results, column
            )
        
        return combined
    
    def weighted_statistics(self, results, column):
        """Calculate weighted statistics across chunks."""
        
        import numpy as np
        
        values = []
        weights = []
        
        for result in results:
            if column in result.get('statistics', {}):
                values.append(result['statistics'][column]['mean'])
                weights.append(result.get('record_count', 1))
        
        if values:
            weighted_mean = np.average(values, weights=weights)
            return {
                'mean': weighted_mean,
                'chunks': len(values),
                'total_weight': sum(weights)
            }
        
        return {}
    
    async def get_file_size(self, file_path):
        """Get file size in bytes."""
        import os
        return os.path.getsize(file_path)
    
    async def optimize_memory(self):
        """Optimize memory usage."""
        import gc
        gc.collect()
        print("🧹 Memory optimization completed")

# Usage example
analyzer = PerformantDataAnalyzer()

# Analyze large dataset with performance optimization
result = await analyzer.analyze_large_dataset(
    '/sandbox/big_data/sales_history.csv',
    {
        'analysis': 'descriptive_statistics',
        'columns': ['revenue', 'quantity', 'profit_margin'],
        'includeCorrelations': True
    }
)

print(f"⚡ Analysis completed in {analyzer.performance_metrics['execution_time']:.2f} seconds")
print(f"💾 Memory usage: {analyzer.performance_metrics['memory_usage'] / 1024 / 1024:.2f} MB")

// End-to-end data analysis workflow
class DataAnalysisPipeline {
    constructor() {
        this.pipeline_id = Date.now().toString();
        this.results = {};
    }
    
    async executeFullPipeline(inputFiles, analysisConfig) {
        console.log(`🚀 Starting data analysis pipeline: ${this.pipeline_id}`);
        
        try {
            // Stage 1: Data ingestion and validation
            const ingestionResults = await this.dataIngestion(inputFiles);
            
            // Stage 2: Data quality assessment
            const qualityResults = await this.dataQualityAssessment(ingestionResults);
            
            // Stage 3: Data cleaning and transformation
            const cleanedData = await this.dataCleaningStage(qualityResults);
            
            // Stage 4: Statistical analysis
            const statisticalResults = await this.statisticalAnalysisStage(cleanedData, analysisConfig);
            
            // Stage 5: Advanced analytics
            const advancedResults = await this.advancedAnalyticsStage(cleanedData, analysisConfig);
            
            // Stage 6: Results compilation and reporting
            const finalReport = await this.generateComprehensiveReport({
                ingestion: ingestionResults,
                quality: qualityResults,
                statistical: statisticalResults,
                advanced: advancedResults
            });
            
            return finalReport;
            
        } catch (error) {
            console.error(`❌ Pipeline failed: ${error.message}`);
            throw error;
        }
    }
    
    async dataIngestion(inputFiles) {
        console.log('📥 Stage 1: Data Ingestion');
        
        const results = [];
        
        for (const file of inputFiles) {
            if (file.path.endsWith('.xlsx')) {
                const excelResult = await xlsxAnalysis({
                    filePath: file.path,
                    operation: 'metadata_extraction',
                    includeAllSheets: true
                });
                results.push({ ...excelResult, source: file.path });
            } else {
                const dataResult = await dataTransformation({
                    inputData: file.path,
                    operation: 'data_profiling',
                    profile: {
                        includeTypes: true,
                        includeStats: true,
                        sampleSize: 10000
                    }
                });
                results.push({ ...dataResult, source: file.path });
            }
        }
        
        return results;
    }
    
    async dataQualityAssessment(ingestionResults) {
        console.log('🔍 Stage 2: Data Quality Assessment');
        
        const qualityReports = [];
        
        for (const result of ingestionResults) {
            const qualityReport = await xlsxAnalysis({
                filePath: result.source,
                operation: 'quality_assessment',
                validation: {
                    checkCompleteness: true,
                    checkConsistency: true,
                    checkAccuracy: true,
                    detectOutliers: true,
                    validateFormats: true
                }
            });
            
            qualityReports.push({
                source: result.source,
                quality: qualityReport
            });
        }
        
        return qualityReports;
    }
    
    async dataCleaningStage(qualityResults) {
        console.log('🧹 Stage 3: Data Cleaning and Transformation');
        
        const cleanedFiles = [];
        
        for (const qualityResult of qualityResults) {
            const cleaningSteps = this.generateCleaningSteps(qualityResult.quality);
            
            const cleanedData = await dataTransformation({
                inputData: qualityResult.source,
                operation: 'comprehensive_cleaning',
                steps: cleaningSteps,
                output: `/sandbox/cleaned/${path.basename(qualityResult.source)}`
            });
            
            cleanedFiles.push(cleanedData.outputPath);
        }
        
        return cleanedFiles;
    }
    
    async statisticalAnalysisStage(cleanedFiles, config) {
        console.log('📊 Stage 4: Statistical Analysis');
        
        const statisticalResults = [];
        
        for (const file of cleanedFiles) {
            const stats = await statisticalAnalysis({
                data: file,
                analysis: 'comprehensive_statistics',
                include: [
                    'descriptive_statistics',
                    'correlation_analysis',
                    'distribution_fitting',
                    'hypothesis_testing'
                ],
                options: config.statistical || {}
            });
            
            statisticalResults.push({
                source: file,
                statistics: stats
            });
        }
        
        return statisticalResults;
    }
    
    async advancedAnalyticsStage(cleanedFiles, config) {
        console.log('🤖 Stage 5: Advanced Analytics');
        
        const advancedResults = [];
        
        for (const file of cleanedFiles) {
            const advanced = await statisticalAnalysis({
                data: file,
                analysis: 'advanced_analytics',
                include: [
                    'regression_analysis',
                    'time_series_analysis',
                    'clustering_analysis',
                    'anomaly_detection'
                ],
                options: config.advanced || {}
            });
            
            advancedResults.push({
                source: file,
                advanced: advanced
            });
        }
        
        return advancedResults;
    }
    
    async generateComprehensiveReport(allResults) {
        console.log('📋 Stage 6: Report Generation');
        
        const report = {
            pipeline_id: this.pipeline_id,
            execution_date: new Date().toISOString(),
            summary: this.generateExecutiveSummary(allResults),
            detailed_results: allResults,
            recommendations: this.generateRecommendations(allResults),
            visualizations: await this.generateVisualizations(allResults)
        };
        
        // Save comprehensive report
        await writeFile({
            path: `/sandbox/reports/analysis_report_${this.pipeline_id}.json`,
            content: JSON.stringify(report, null, 2)
        });
        
        return report;
    }
    
    generateCleaningSteps(qualityReport) {
        const steps = [];
        
        if (qualityReport.completeness && qualityReport.completeness.missingPercentage > 0.1) {
            steps.push({
                type: 'handle_missing',
                strategy: 'smart_imputation'
            });
        }
        
        if (qualityReport.outliers && qualityReport.outliers.count > 0) {
            steps.push({
                type: 'outlier_treatment',
                method: 'iqr',
                action: 'cap'
            });
        }
        
        return steps;
    }
    
    generateExecutiveSummary(results) {
        return {
            files_processed: results.statistical.length,
            data_quality_score: this.calculateQualityScore(results.quality),
            key_insights: this.extractKeyInsights(results.statistical),
            anomalies_detected: this.countAnomalies(results.advanced)
        };
    }
    
    generateRecommendations(results) {
        const recommendations = [];
        
        // Data quality recommendations
        for (const quality of results.quality) {
            if (quality.quality.completeness?.missingPercentage > 0.2) {
                recommendations.push({
                    type: 'data_quality',
                    priority: 'high',
                    message: `High missing data rate detected in ${quality.source}. Consider data collection improvements.`
                });
            }
        }
        
        return recommendations;
    }
    
    async generateVisualizations(results) {
        // Generate visualization configs that could be used with charting tools
        const visualizations = [];
        
        for (const stat of results.statistical) {
            if (stat.statistics.correlation_matrix) {
                visualizations.push({
                    type: 'correlation_heatmap',
                    data: stat.statistics.correlation_matrix,
                    title: `Correlation Matrix - ${path.basename(stat.source)}`
                });
            }
        }
        
        return visualizations;
    }
    
    calculateQualityScore(qualityResults) {
        // Simple quality score calculation
        let totalScore = 0;
        
        for (const result of qualityResults) {
            const completeness = 1 - (result.quality.completeness?.missingPercentage || 0);
            const accuracy = 1 - (result.quality.accuracy?.errorRate || 0);
            const score = (completeness + accuracy) / 2;
            totalScore += score;
        }
        
        return totalScore / qualityResults.length;
    }
    
    extractKeyInsights(statisticalResults) {
        const insights = [];
        
        for (const result of statisticalResults) {
            if (result.statistics.correlation_analysis) {
                const strongCorrelations = result.statistics.correlation_analysis.strong_correlations || [];
                insights.push(...strongCorrelations.map(corr => 
                    `Strong correlation detected between ${corr.variable1} and ${corr.variable2} (${corr.coefficient})`
                ));
            }
        }
        
        return insights;
    }
    
    countAnomalies(advancedResults) {
        let totalAnomalies = 0;
        
        for (const result of advancedResults) {
            if (result.advanced.anomaly_detection) {
                totalAnomalies += result.advanced.anomaly_detection.count || 0;
            }
        }
        
        return totalAnomalies;
    }
}

// Execute comprehensive analysis pipeline
const pipeline = new DataAnalysisPipeline();
const inputFiles = [
    { path: '/sandbox/data/sales_data.xlsx' },
    { path: '/sandbox/data/customer_data.csv' },
    { path: '/sandbox/data/product_data.xlsx' }
];

const analysisConfig = {
    statistical: {
        includeCorrelations: true,
        confidenceLevel: 0.95,
        hypothesisTests: ['normality', 'independence']
    },
    advanced: {
        regressionType: 'multiple_linear',
        clusteringMethod: 'kmeans',
        anomalyThreshold: 2.5
    }
};

const comprehensiveReport = await pipeline.executeFullPipeline(inputFiles, analysisConfig);
console.log('✅ Comprehensive data analysis completed!');