112 lines
3.3 KiB
Bash
Executable File
112 lines
3.3 KiB
Bash
Executable File
#!/bin/bash
|
|
# Quick Start Script for Proxy Scraping Service
|
|
# This script performs initial setup and starts the service
|
|
|
|
set -e
|
|
|
|
echo "=================================================="
|
|
echo "Proxy Scraping Service - Quick Start"
|
|
echo "=================================================="
|
|
echo ""
|
|
|
|
# Check if Docker is running
|
|
if ! docker info > /dev/null 2>&1; then
|
|
echo "Error: Docker is not running. Please start Docker and try again."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if docker-compose is available
|
|
if ! command -v docker-compose &> /dev/null; then
|
|
echo "Error: docker-compose is not installed."
|
|
exit 1
|
|
fi
|
|
|
|
# Step 1: Create .env file if it doesn't exist
|
|
if [ ! -f .env ]; then
|
|
echo "Step 1: Creating .env file from template..."
|
|
cp .env.example .env
|
|
echo " .env file created successfully"
|
|
echo ""
|
|
echo "IMPORTANT: Please edit .env and set secure passwords!"
|
|
echo " nano .env"
|
|
echo ""
|
|
read -p "Press Enter to continue after editing .env (or skip if using defaults)..."
|
|
else
|
|
echo "Step 1: .env file already exists, skipping..."
|
|
fi
|
|
echo ""
|
|
|
|
# Step 2: Check proxies.txt
|
|
if [ ! -f root/proxy/proxies.txt ]; then
|
|
echo "Error: root/proxy/proxies.txt not found!"
|
|
exit 1
|
|
else
|
|
echo "Step 2: Checking proxies.txt..."
|
|
url_count=$(grep -v '^#' root/proxy/proxies.txt | grep -v '^$' | wc -l | tr -d ' ')
|
|
echo " Found $url_count URLs to scrape"
|
|
fi
|
|
echo ""
|
|
|
|
# Step 3: Build Docker images
|
|
echo "Step 3: Building Docker images (this may take a few minutes)..."
|
|
docker-compose build
|
|
echo " Images built successfully"
|
|
echo ""
|
|
|
|
# Step 4: Start services
|
|
echo "Step 4: Starting services..."
|
|
docker-compose up -d
|
|
echo " Services started successfully"
|
|
echo ""
|
|
|
|
# Step 5: Wait for PostgreSQL to be ready
|
|
echo "Step 5: Waiting for PostgreSQL to be ready..."
|
|
timeout=60
|
|
elapsed=0
|
|
until docker-compose exec -T postgres pg_isready -U postgres > /dev/null 2>&1; do
|
|
if [ $elapsed -ge $timeout ]; then
|
|
echo " Error: PostgreSQL did not become ready in time"
|
|
exit 1
|
|
fi
|
|
echo " Waiting for PostgreSQL... ($elapsed/$timeout seconds)"
|
|
sleep 2
|
|
elapsed=$((elapsed + 2))
|
|
done
|
|
echo " PostgreSQL is ready"
|
|
echo ""
|
|
|
|
# Step 6: Show status
|
|
echo "Step 6: Service status..."
|
|
docker-compose ps
|
|
echo ""
|
|
|
|
# Step 7: Instructions
|
|
echo "=================================================="
|
|
echo "Setup Complete!"
|
|
echo "=================================================="
|
|
echo ""
|
|
echo "Your proxy scraping service is now running."
|
|
echo ""
|
|
echo "Useful commands:"
|
|
echo " docker-compose logs -f proxy-scraper # View logs"
|
|
echo " make logs # View logs (if make installed)"
|
|
echo " make immediate # Run scraping now"
|
|
echo " make stats # Show database stats"
|
|
echo " docker-compose down # Stop services"
|
|
echo ""
|
|
echo "The service will automatically scrape proxies daily between 2-4 AM UTC."
|
|
echo ""
|
|
echo "To run an immediate scraping job:"
|
|
echo " docker-compose exec proxy-scraper python src/main.py --immediate"
|
|
echo ""
|
|
echo "Access pgAdmin (database management):"
|
|
echo " docker-compose --profile admin up -d"
|
|
echo " Then open: http://localhost:5050"
|
|
echo ""
|
|
|
|
read -p "Do you want to view the logs now? (y/n) " -n 1 -r
|
|
echo
|
|
if [[ $REPLY =~ ^[Yy]$ ]]; then
|
|
docker-compose logs -f proxy-scraper
|
|
fi
|