// Ethical scraping configuration
const ethicalConfig = {
    rateLimiting: {
        requestDelay: 3000,        // 3 seconds between requests
        maxConcurrent: 2,          // Max 2 simultaneous requests per domain
        respectRetryAfter: true,   // Honor server retry-after headers
        exponentialBackoff: true,  // Increase delays on errors
        dailyQuota: 1000          // Max 1000 requests per day per domain
    },
    compliance: {
        checkRobotsTxt: true,      // Always check robots.txt
        respectNoIndex: true,      // Skip pages with noindex directive
        userAgent: "Axellero Web Scraper 1.0",
        contactInfo: "admin@example.com"
    },
    contentFiltering: {
        blockPersonalData: true,   // Filter out PII
        copyrightDetection: true,  // Check for copyrighted content
        adultContentFilter: true,  // Skip adult content
        malwareCheck: true        // Scan for malicious content
    }
};

// Apply configuration to scraping
const result = await webScraping({
    url: "https://example.com",
    config: ethicalConfig,
    extractors: dataExtractors
});

# Complete web research workflow
async def comprehensive_web_research(research_topic):
    """Conduct comprehensive research using web tools."""
    
    # 1. Multi-engine web search
    print(f"🔍 Researching: {research_topic}")
    
    search_results = await webSearch({
        'query': research_topic,
        'engines': ['google', 'bing', 'academic'],
        'maxResults': 100,
        'filters': {
            'dateRange': 'past_2_years',
            'contentType': ['article', 'research', 'blog'],
            'language': 'en'
        }
    })
    
    # 2. Filter and rank results
    relevant_sources = []
    for result in search_results.get('results', []):
        if result['relevanceScore'] > 0.7:
            relevant_sources.append(result)
    
    print(f"📊 Found {len(relevant_sources)} relevant sources")
    
    # 3. Extract content from top sources
    extracted_content = []
    
    for source in relevant_sources[:20]:  # Process top 20 sources
        try:
            content = await webScraping({
                'url': source['url'],
                'extractors': [
                    {
                        'name': 'main_content',
                        'selector': 'article, .content, .post, main',
                        'fields': {
                            'title': 'h1, h2',
                            'content': 'p, div.text',
                            'author': '.author, .byline',
                            'date': '.date, .published'
                        }
                    }
                ],
                'config': {
                    'rateLimiting': {'requestDelay': 2000},
                    'compliance': {'checkRobotsTxt': True}
                }
            })
            
            if content['success']:
                extracted_content.append({
                    'source': source,
                    'content': content['data']
                })
                
        except Exception as e:
            print(f"⚠️ Failed to extract from {source['url']}: {e}")
    
    # 4. Collect supporting images
    image_results = await imageSearch({
        'query': f"{research_topic} infographic diagram",
        'filters': {
            'license': ['creative_commons', 'public_domain'],
            'format': ['png', 'svg', 'jpg']
        },
        'maxResults': 10
    })
    
    # 5. Organize and structure findings
    research_data = {
        'topic': research_topic,
        'search_summary': {
            'total_results': len(search_results.get('results', [])),
            'relevant_sources': len(relevant_sources),
            'extracted_articles': len(extracted_content)
        },
        'sources': extracted_content,
        'supporting_images': image_results.get('images', []),
        'research_date': datetime.now().isoformat()
    }
    
    # 6. Save research data
    await writeFile({
        'path': f'/sandbox/research/{research_topic}_research.json',
        'content': json.dumps(research_data, indent=2)
    })
    
    print(f"✅ Research completed. Data saved to sandbox.")
    return research_data

# Execute research workflow
research_results = await comprehensive_web_research("sustainable energy technologies")

// Competitive analysis using web tools
class CompetitiveAnalyzer {
    constructor() {
        this.competitors = [];
        this.analysisData = {};
    }
    
    async analyzeCompetitors(industry, targetCompanies) {
        console.log(`🏢 Analyzing ${targetCompanies.length} competitors in ${industry}`);
        
        for (const company of targetCompanies) {
            const analysis = await this.analyzeCompany(company);
            this.analysisData[company] = analysis;
        }
        
        return this.generateCompetitiveReport();
    }
    
    async analyzeCompany(companyName) {
        // 1. Search for company information
        const companySearch = await webSearch({
            query: `${companyName} company profile products services`,
            engines: ['google', 'bing'],
            maxResults: 50,
            filters: {
                domain: [
                    'bloomberg.com', 'reuters.com', 'crunchbase.com',
                    'linkedin.com', 'glassdoor.com'
                ]
            }
        });
        
        // 2. Scrape company website
        const websiteData = await this.scrapeCompanyWebsite(companyName);
        
        // 3. Collect product images and marketing materials
        const marketingImages = await imageSearch({
            query: `${companyName} products marketing materials`,
            filters: {
                license: ['any'], // For analysis purposes
                format: ['jpg', 'png']
            },
            maxResults: 15
        });
        
        // 4. Analyze news and press coverage
        const newsAnalysis = await this.analyzeNews(companyName);
        
        return {
            company: companyName,
            searchResults: companySearch,
            websiteData: websiteData,
            marketingMaterials: marketingImages,
            newsAnalysis: newsAnalysis,
            analysisDate: new Date().toISOString()
        };
    }
    
    async scrapeCompanyWebsite(companyName) {
        // Try to find the company's main website
        const websiteSearch = await webSearch({
            query: `${companyName} official website`,
            maxResults: 5
        });
        
        if (!websiteSearch.results || websiteSearch.results.length === 0) {
            return null;
        }
        
        const mainWebsite = websiteSearch.results[0].url;
        
        try {
            const websiteContent = await webScraping({
                url: mainWebsite,
                extractors: [
                    {
                        name: 'navigation',
                        selector: 'nav, .navigation, .menu',
                        fields: {
                            links: 'a',
                            sections: 'li, .nav-item'
                        }
                    },
                    {
                        name: 'products',
                        selector: '.product, .service, .solution',
                        fields: {
                            title: 'h1, h2, h3',
                            description: 'p, .description',
                            features: 'ul li, .features li'
                        }
                    },
                    {
                        name: 'about',
                        selector: '.about, #about, .company-info',
                        fields: {
                            description: 'p',
                            mission: '.mission, .vision',
                            history: '.history, .timeline'
                        }
                    }
                ],
                config: {
                    rateLimiting: { requestDelay: 3000 },
                    compliance: { checkRobotsTxt: true }
                }
            });
            
            return websiteContent.data;
            
        } catch (error) {
            console.warn(`⚠️ Could not scrape ${mainWebsite}: ${error.message}`);
            return null;
        }
    }
    
    async analyzeNews(companyName) {
        const newsSearch = await webSearch({
            query: `"${companyName}" news press release funding`,
            engines: ['google', 'bing'],
            filters: {
                dateRange: 'past_year',
                contentType: ['news', 'article'],
                domain: [
                    'techcrunch.com', 'venturebeat.com', 'businesswire.com',
                    'prnewswire.com', 'reuters.com', 'bloomberg.com'
                ]
            },
            maxResults: 30
        });
        
        // Categorize news by sentiment and topic
        const newsCategories = {
            funding: [],
            product_launches: [],
            partnerships: [],
            leadership: [],
            other: []
        };
        
        for (const article of newsSearch.results || []) {
            const title = article.title.toLowerCase();
            
            if (title.includes('funding') || title.includes('investment') || title.includes('raised')) {
                newsCategories.funding.push(article);
            } else if (title.includes('launch') || title.includes('release') || title.includes('product')) {
                newsCategories.product_launches.push(article);
            } else if (title.includes('partnership') || title.includes('collaboration')) {
                newsCategories.partnerships.push(article);
            } else if (title.includes('ceo') || title.includes('leadership') || title.includes('executive')) {
                newsCategories.leadership.push(article);
            } else {
                newsCategories.other.push(article);
            }
        }
        
        return newsCategories;
    }
    
    generateCompetitiveReport() {
        const report = {
            summary: {
                companiesAnalyzed: Object.keys(this.analysisData).length,
                analysisDate: new Date().toISOString(),
                methodology: "Web search, scraping, and image analysis"
            },
            competitors: this.analysisData,
            insights: this.generateInsights()
        };
        
        return report;
    }
    
    generateInsights() {
        // Analyze patterns across competitors
        const insights = {
            commonProducts: this.findCommonProducts(),
            marketingTrends: this.analyzeMarketingTrends(),
            newsPatterns: this.analyzeNewsPatterns()
        };
        
        return insights;
    }
    
    findCommonProducts() {
        // Implementation for finding common product categories
        return {};
    }
    
    analyzeMarketingTrends() {
        // Implementation for analyzing marketing materials
        return {};
    }
    
    analyzeNewsPatterns() {
        // Implementation for analyzing news patterns
        return {};
    }
}

// Usage
const analyzer = new CompetitiveAnalyzer();
const competitorList = ['Company A', 'Company B', 'Company C'];
const analysis = await analyzer.analyzeCompetitors('SaaS', competitorList);

# Web data collection and file management workflow
async def web_to_file_workflow(research_topics):
    """Collect web data and organize in file system."""
    
    for topic in research_topics:
        print(f"📁 Processing topic: {topic}")
        
        # Create directory for topic
        topic_dir = f"/sandbox/research/{topic.replace(' ', '_')}"
        await createDirectory({
            'path': topic_dir,
            'recursive': True
        })
        
        # 1. Web search and save results
        search_results = await webSearch({
            'query': topic,
            'maxResults': 50
        })
        
        await writeFile({
            'path': f"{topic_dir}/search_results.json",
            'content': json.dumps(search_results, indent=2)
        })
        
        # 2. Collect images and save metadata
        images = await imageSearch({
            'query': topic,
            'maxResults': 10
        })
        
        await writeFile({
            'path': f"{topic_dir}/images_metadata.json",
            'content': json.dumps(images, indent=2)
        })
        
        # 3. Scrape top articles and save content
        for i, result in enumerate(search_results['results'][:5]):
            try:
                content = await webScraping({
                    'url': result['url']
                })
                
                if content['success']:
                    filename = f"article_{i+1}_{result['title'][:50]}.json"
                    filename = "".join(c for c in filename if c.isalnum() or c in ('_', '-', '.'))
                    
                    await writeFile({
                        'path': f"{topic_dir}/{filename}",
                        'content': json.dumps(content, indent=2)
                    })
                    
            except Exception as e:
                print(f"⚠️ Failed to scrape {result['url']}: {e}")
    
    # Create summary report
    all_files = await listFiles({
        'path': '/sandbox/research/',
        'recursive': True
    })
    
    summary = {
        'topics_researched': len(research_topics),
        'total_files_created': len(all_files),
        'research_date': datetime.now().isoformat()
    }
    
    await writeFile({
        'path': '/sandbox/research/summary_report.json',
        'content': json.dumps(summary, indent=2)
    })
    
    return summary

# Execute workflow
topics = ["artificial intelligence", "blockchain technology", "renewable energy"]
summary = await web_to_file_workflow(topics)

// Comprehensive error handling for web operations
class WebOperationManager {
    constructor() {
        this.retryAttempts = 3;
        this.retryDelay = 1000;
        this.operationLog = [];
    }
    
    async safeWebSearch(params) {
        return this.executeWithRetry('webSearch', webSearch, params);
    }
    
    async safeWebScraping(params) {
        return this.executeWithRetry('webScraping', webScraping, params);
    }
    
    async safeImageSearch(params) {
        return this.executeWithRetry('imageSearch', imageSearch, params);
    }
    
    async executeWithRetry(operationType, operation, params) {
        let lastError = null;
        
        for (let attempt = 1; attempt <= this.retryAttempts; attempt++) {
            try {
                const result = await operation(params);
                
                this.logOperation(operationType, 'success', {
                    attempt,
                    params: this.sanitizeParams(params),
                    result: this.summarizeResult(result)
                });
                
                return result;
                
            } catch (error) {
                lastError = error;
                
                this.logOperation(operationType, 'error', {
                    attempt,
                    error: error.message,
                    params: this.sanitizeParams(params)
                });
                
                if (attempt < this.retryAttempts) {
                    const delay = this.calculateDelay(attempt);
                    console.log(`⏳ Retrying ${operationType} in ${delay}ms (attempt ${attempt + 1}/${this.retryAttempts})`);
                    await new Promise(resolve => setTimeout(resolve, delay));
                } else {
                    console.error(`❌ ${operationType} failed after ${this.retryAttempts} attempts`);
                }
            }
        }
        
        throw new Error(`Operation ${operationType} failed: ${lastError.message}`);
    }
    
    calculateDelay(attempt) {
        // Exponential backoff with jitter
        const baseDelay = this.retryDelay * Math.pow(2, attempt - 1);
        const jitter = Math.random() * 1000;
        return baseDelay + jitter;
    }
    
    logOperation(type, status, details) {
        const logEntry = {
            timestamp: new Date().toISOString(),
            operation: type,
            status,
            ...details
        };
        
        this.operationLog.push(logEntry);
        
        // Keep only last 100 operations
        if (this.operationLog.length > 100) {
            this.operationLog.shift();
        }
    }
    
    sanitizeParams(params) {
        // Remove sensitive information from logs
        const sanitized = { ...params };
        delete sanitized.apiKeys;
        delete sanitized.credentials;
        return sanitized;
    }
    
    summarizeResult(result) {
        // Create summary without full data
        if (result.results) {
            return { resultCount: result.results.length };
        }
        if (result.images) {
            return { imageCount: result.images.length };
        }
        if (result.data) {
            return { dataExtracted: true };
        }
        return { status: 'completed' };
    }
    
    getOperationStats() {
        const stats = {
            totalOperations: this.operationLog.length,
            successRate: 0,
            errorsByType: {},
            averageAttempts: 0
        };
        
        let successCount = 0;
        let totalAttempts = 0;
        
        for (const log of this.operationLog) {
            if (log.status === 'success') {
                successCount++;
            } else {
                stats.errorsByType[log.operation] = (stats.errorsByType[log.operation] || 0) + 1;
            }
            totalAttempts += log.attempt || 1;
        }
        
        stats.successRate = (successCount / this.operationLog.length) * 100;
        stats.averageAttempts = totalAttempts / this.operationLog.length;
        
        return stats;
    }
}

// Usage with error handling
const webManager = new WebOperationManager();

try {
    // Safe web operations with automatic retry
    const searchResults = await webManager.safeWebSearch({
        query: "machine learning",
        maxResults: 20
    });
    
    const scrapingResults = await webManager.safeWebScraping({
        url: "https://example.com/data"
    });
    
    // Monitor operation statistics
    const stats = webManager.getOperationStats();
    console.log(`📊 Success Rate: ${stats.successRate.toFixed(2)}%`);
    
} catch (error) {
    console.error('🚨 Critical error in web operations:', error.message);
}