Разработка бота-парсера цен конкурентов по расписанию

Разработка бота-парсера цен конкурентов по расписанию

Мониторинг цен конкурентов — основа динамического ценообразования. Бот собирает цены по расписанию, хранит историю изменений и может автоматически корректировать ваши цены по заданным правилам.

Архитектура системы мониторинга

Scheduler
  → ScrapeCompetitorPrices Job (per competitor)
      → CompetitorScraper (HTTP/Playwright)
      → PriceNormalizer
      → PriceHistoryRepository (INSERT)
      → PriceChangeDetector
          → AlertDispatcher (если изменение > порога)
          → RepricingEngine (если включён autoprice)

Модель данных

// database/migrations/create_competitor_prices_table.php
Schema::create('competitor_prices', function (Blueprint $table) {
    $table->id();
    $table->foreignId('product_id')->constrained();
    $table->foreignId('competitor_id')->constrained();
    $table->string('competitor_sku')->nullable();
    $table->string('competitor_url');
    $table->decimal('price', 12, 2);
    $table->decimal('sale_price', 12, 2)->nullable();
    $table->boolean('in_stock')->default(true);
    $table->timestamp('scraped_at');
    $table->timestamps();

    $table->index(['product_id', 'competitor_id', 'scraped_at']);
});

// История для графиков и аналитики
Schema::create('competitor_price_history', function (Blueprint $table) {
    $table->id();
    $table->foreignId('product_id')->constrained();
    $table->foreignId('competitor_id')->constrained();
    $table->decimal('price', 12, 2);
    $table->boolean('in_stock');
    $table->date('recorded_date');
    $table->timestamps();

    $table->unique(['product_id', 'competitor_id', 'recorded_date']);
});

Конфигурация конкурентов

// config/competitors.php
return [
    'competitor_a' => [
        'name'       => 'Shop A',
        'base_url'   => 'https://shop-a.ru',
        'selectors'  => [
            'price'    => '.current-price',
            'in_stock' => '.in-stock-badge',
        ],
        'price_regex'    => '/[\d\s]+/',
        'url_pattern'    => 'https://shop-a.ru/product/{sku}',
        'requires_js'    => false,
        'request_delay'  => [1000, 3000], // ms
    ],

    'wildberries' => [
        'name'       => 'Wildberries',
        'type'       => 'api',
        'scraper'    => WildberriesPriceScraper::class,
        'request_delay' => [500, 1500],
    ],
];

Базовый скрапер цен

// app/Services/PriceMonitor/CompetitorPriceScraper.php
class CompetitorPriceScraper
{
    public function __construct(
        private Client $httpClient,
        private array $config
    ) {}

    public function scrapePrice(string $url): ?PriceData
    {
        try {
            $html = $this->fetch($url);
            $crawler = new Crawler($html);

            $priceText = $crawler->filter($this->config['selectors']['price'])
                ->first()
                ->text('');

            $price = $this->extractPrice($priceText);
            if ($price === null) return null;

            $salePrice = null;
            if (!empty($this->config['selectors']['sale_price'])) {
                $salePriceText = $crawler->filter($this->config['selectors']['sale_price'])
                    ->first()->text('');
                $salePrice = $this->extractPrice($salePriceText);
            }

            $inStock = true;
            if (!empty($this->config['selectors']['in_stock'])) {
                $inStock = $crawler->filter($this->config['selectors']['in_stock'])->count() > 0;
            }

            return new PriceData(
                price: $price,
                salePrice: $salePrice,
                inStock: $inStock,
            );
        } catch (\Exception $e) {
            Log::warning("Price scrape failed for {$url}: {$e->getMessage()}");
            return null;
        }
    }

    private function extractPrice(string $text): ?float
    {
        $cleaned = preg_replace('/[^\d,.]/', '', str_replace(' ', '', $text));
        $cleaned = str_replace(',', '.', $cleaned);
        return is_numeric($cleaned) ? (float) $cleaned : null;
    }
}

Job с детектором изменений

// app/Jobs/MonitorCompetitorPrice.php
class MonitorCompetitorPrice implements ShouldQueue
{
    public int $tries = 3;
    public array $backoff = [60, 120, 300];

    public function __construct(
        private int $productId,
        private int $competitorId,
        private string $competitorUrl
    ) {}

    public function handle(
        CompetitorPriceScraper $scraper,
        PriceChangeDetector $detector,
        RepricingEngine $repricer
    ): void {
        $config = Competitor::find($this->competitorId)->config;
        $priceData = $scraper->scrapePrice($this->competitorUrl);

        if ($priceData === null) return;

        // Сохранение текущей цены
        $record = CompetitorPrice::updateOrCreate(
            [
                'product_id'    => $this->productId,
                'competitor_id' => $this->competitorId,
            ],
            [
                'price'        => $priceData->price,
                'sale_price'   => $priceData->salePrice,
                'in_stock'     => $priceData->inStock,
                'scraped_at'   => now(),
            ]
        );

        // Сохранение в историю (один раз в день)
        CompetitorPriceHistory::firstOrCreate(
            [
                'product_id'    => $this->productId,
                'competitor_id' => $this->competitorId,
                'recorded_date' => today(),
            ],
            [
                'price'    => $priceData->price,
                'in_stock' => $priceData->inStock,
            ]
        );

        // Детект изменений
        if ($record->wasChanged('price')) {
            $change = $detector->analyze($record);

            if ($change->isSignificant()) {
                Notification::route('mail', config('monitoring.alert_email'))
                    ->notify(new CompetitorPriceChangedNotification($record, $change));
            }

            // Автоперейценка
            if (config('repricing.enabled')) {
                $repricer->recalculate($this->productId);
            }
        }
    }
}

Движок автоматического перейценивания

// app/Services/PriceMonitor/RepricingEngine.php
class RepricingEngine
{
    public function recalculate(int $productId): void
    {
        $product = Product::with('competitorPrices', 'repricingRule')->findOrFail($productId);
        $rule = $product->repricingRule;

        if (!$rule || !$rule->is_active) return;

        $competitorPrices = $product->competitorPrices
            ->where('in_stock', true)
            ->pluck('price');

        if ($competitorPrices->isEmpty()) return;

        $newPrice = match ($rule->strategy) {
            'beat_lowest'  => $competitorPrices->min() - $rule->delta,
            'match_lowest' => $competitorPrices->min(),
            'beat_average' => $competitorPrices->avg() - $rule->delta,
            'percentile'   => $this->percentile($competitorPrices, $rule->percentile),
            default        => null,
        };

        if (!$newPrice) return;

        // Применяем ограничения
        $newPrice = max($newPrice, $rule->min_price ?? 0);
        $newPrice = min($newPrice, $rule->max_price ?? PHP_FLOAT_MAX);

        // Не перейцениваем, если изменение < 0.5%
        $currentPrice = $product->price;
        if (abs($newPrice - $currentPrice) / $currentPrice < 0.005) return;

        $product->update(['price' => round($newPrice, 2)]);

        Log::info("Repriced product #{$productId}", [
            'old_price' => $currentPrice,
            'new_price' => $newPrice,
            'strategy'  => $rule->strategy,
        ]);
    }
}

Расписание мониторинга

// app/Console/Kernel.php
protected function schedule(Schedule $schedule): void
{
    // Высокоприоритетные товары — каждые 2 часа
    $schedule->command('monitor:prices --priority=high')
        ->everyTwoHours()->withoutOverlapping();

    // Все товары — раз в день, ночью
    $schedule->command('monitor:prices --all')
        ->dailyAt('02:00')->withoutOverlapping();

    // Агрегация истории цен
    $schedule->command('prices:aggregate-history')
        ->dailyAt('23:50');
}

Дашборд мониторинга

Ключевые метрики для отображения:

Срок разработки: мониторинг 1 конкурента + история + алерты — 4-6 рабочих дней. Система с автоперейценкой и дашбордом для 5+ конкурентов — 10-14 дней.