Разработка бота-парсера отзывов на товары с внешних площадок

Разработка бота-парсера отзывов на товары с внешних площадок

Отзывы с маркетплейсов и агрегаторов — ценный контент для карточки товара: повышают доверие, добавляют ключевые слова в UGC, влияют на SEO через structured data. Парсер собирает отзывы, нормализует их и импортирует в базу магазина.

Источники и методы доступа

Wildberries: парсинг через JSON API

# scraper/reviews/wildberries.py
import httpx
import asyncio
from dataclasses import dataclass
from typing import Optional

@dataclass
class Review:
    external_id: str
    product_nm_id: int
    author: str
    rating: int
    text: str
    pros: Optional[str]
    cons: Optional[str]
    date: str
    photos: list[str]
    helpful_count: int

class WildberriesReviewScraper:
    REVIEWS_URL = "https://feedbacks2.wb.ru/feedbacks/v2/{nm_id}"

    def __init__(self):
        self.client = httpx.AsyncClient(
            headers={
                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
                "Origin": "https://www.wildberries.ru",
                "Referer": "https://www.wildberries.ru/",
            }
        )

    async def get_reviews(self, nm_id: int, take: int = 100) -> list[Review]:
        all_reviews = []
        skip = 0

        while True:
            url = self.REVIEWS_URL.format(nm_id=nm_id)
            params = {
                "immt": nm_id,
                "skip": skip,
                "take": take,
                "order": "dateDesc",
            }

            resp = await self.client.get(url, params=params)
            resp.raise_for_status()

            data = resp.json()
            feedbacks = data.get("feedbacks", [])

            if not feedbacks:
                break

            for fb in feedbacks:
                all_reviews.append(self._normalize(nm_id, fb))

            skip += take
            await asyncio.sleep(1.0)

            # Ограничение: не более 1000 отзывов за сессию
            if skip >= 1000:
                break

        return all_reviews

    def _normalize(self, nm_id: int, raw: dict) -> Review:
        photos = []
        for photo in raw.get("photos", []):
            if full_url := photo.get("fullSize"):
                photos.append(full_url)

        return Review(
            external_id=raw.get("id", ""),
            product_nm_id=nm_id,
            author=raw.get("wbUserDetails", {}).get("name", "Покупатель"),
            rating=raw.get("productValuation", 0),
            text=raw.get("text", ""),
            pros=raw.get("pros"),
            cons=raw.get("cons"),
            date=raw.get("createdDate", ""),
            photos=photos,
            helpful_count=raw.get("feedbackValuation", 0),
        )

Парсинг HTML-отзывов (iHerb, Otzovik)

// app/Services/ReviewScraper/HtmlReviewScraper.php
class HtmlReviewScraper
{
    public function scrapeIherb(string $productUrl, int $pages = 5): array
    {
        $reviews = [];

        for ($page = 1; $page <= $pages; $page++) {
            $html = $this->fetch("{$productUrl}?p={$page}&is=1&s=6");
            $crawler = new Crawler($html);

            $items = $crawler->filter('[itemprop="review"]');
            if (!$items->count()) break;

            $items->each(function (Crawler $node) use (&$reviews) {
                $reviews[] = [
                    'external_id' => $node->attr('data-review-id'),
                    'author'      => trim($node->filter('[itemprop="author"]')->text('')),
                    'rating'      => (int) $node->filter('[itemprop="ratingValue"]')->attr('content'),
                    'date'        => $node->filter('[itemprop="datePublished"]')->attr('content'),
                    'title'       => trim($node->filter('[itemprop="name"]')->text('')),
                    'text'        => trim($node->filter('[itemprop="reviewBody"]')->text('')),
                    'helpful'     => (int) $node->filter('.helpful-yes')->text('0'),
                    'verified'    => $node->filter('.verified-buyer')->count() > 0,
                ];
            });

            sleep(rand(2, 4));
        }

        return $reviews;
    }
}

Laravel Job с обработкой дублей

// app/Jobs/ImportProductReviews.php
class ImportProductReviews implements ShouldQueue
{
    public int $tries = 3;
    public int $backoff = 120;

    public function handle(ReviewImportService $service): void
    {
        $mapping = ProductReviewMapping::where('product_id', $this->productId)
            ->where('source', $this->source)
            ->firstOrFail();

        $reviews = $this->scrape($mapping->external_id);

        $imported = 0;
        $skipped = 0;

        foreach ($reviews as $reviewData) {
            // Дедупликация по external_id + source
            $exists = ProductReview::where([
                'source'      => $this->source,
                'external_id' => $reviewData['external_id'],
            ])->exists();

            if ($exists) {
                $skipped++;
                continue;
            }

            $service->import($this->productId, $this->source, $reviewData);
            $imported++;
        }

        Log::info("Reviews imported", [
            'product_id' => $this->productId,
            'source'     => $this->source,
            'imported'   => $imported,
            'skipped'    => $skipped,
        ]);
    }
}

Модерация и фильтрация

// app/Services/ReviewImportService.php
class ReviewImportService
{
    // Стоп-слова: спам, реклама, ненормативная лексика
    private array $stopWords = ['купите', 'скидка', 'промокод', 'vk.com', 't.me'];

    public function import(int $productId, string $source, array $data): ?ProductReview
    {
        // Фильтрация слишком коротких отзывов
        if (mb_strlen($data['text']) < 20) return null;

        // Фильтрация стоп-слов (спам)
        foreach ($this->stopWords as $word) {
            if (mb_stripos($data['text'], $word) !== false) return null;
        }

        return ProductReview::create([
            'product_id'  => $productId,
            'source'      => $source,
            'external_id' => $data['external_id'],
            'author'      => $this->anonymizeAuthor($data['author']),
            'rating'      => max(1, min(5, (int) $data['rating'])),
            'text'        => $this->sanitize($data['text']),
            'pros'        => $this->sanitize($data['pros'] ?? ''),
            'cons'        => $this->sanitize($data['cons'] ?? ''),
            'date'        => $data['date'],
            'is_verified' => $data['verified'] ?? false,
            'helpful'     => $data['helpful'] ?? 0,
            'status'      => 'pending', // Требует проверки модератором
        ]);
    }

    private function anonymizeAuthor(string $name): string
    {
        // "Иван Петров" → "Иван П."
        $parts = explode(' ', trim($name));
        if (count($parts) >= 2) {
            return $parts[0] . ' ' . mb_substr($parts[1], 0, 1) . '.';
        }
        return $name ?: 'Покупатель';
    }

    private function sanitize(string $text): string
    {
        return strip_tags(trim($text));
    }
}

Structured data для SEO

После импорта отзывы публикуются в JSON-LD на странице товара:

// app/Http/Controllers/ProductController.php
public function show(string $slug): Response
{
    $product = Product::withReviews()->findBySlug($slug);

    $reviewSchema = $product->reviews->map(fn($r) => [
        '@type'         => 'Review',
        'author'        => ['@type' => 'Person', 'name' => $r->author],
        'datePublished' => $r->date,
        'reviewBody'    => $r->text,
        'reviewRating'  => [
            '@type'       => 'Rating',
            'ratingValue' => $r->rating,
            'bestRating'  => 5,
        ],
    ]);

    $aggregateRating = [
        '@type'       => 'AggregateRating',
        'ratingValue' => round($product->reviews->avg('rating'), 1),
        'reviewCount' => $product->reviews->count(),
    ];
}

Расписание обновлений

// Новые отзывы — каждый день
$schedule->command('reviews:sync --source=wildberries')
    ->dailyAt('06:00')->withoutOverlapping();

// Популярные товары — чаще
$schedule->command('reviews:sync --source=wildberries --top-products')
    ->everyFourHours()->withoutOverlapping();

Срок разработки: парсер одной площадки с модерацией и structured data — 3-5 рабочих дней.